diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index ada5fb0fe64dc2..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:40 - -ENV CC=clang - -ENV WASI_SDK_VERSION=24 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=22.0.0 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..64c85c1101e6e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 680f2ed5be031a..221008717b29b1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -207,7 +207,6 @@ Doc/c-api/stable.rst @encukou **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith @@ -281,3 +280,5 @@ Lib/test/test_configparser.py @jaraco # Doc sections Doc/reference/ @willingc + +**/*weakref* @kumaraditya303 \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index c14d7cf2599d4c..6d73f7cae5c0ae 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -32,6 +32,7 @@ body: - "3.10" - "3.11" - "3.12" + - "3.13" - "CPython main branch" validations: required: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5f6fd47e7367b..ec7904c2e2cc73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -195,13 +195,14 @@ jobs: build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' strategy: fail-fast: false matrix: + os: [ubuntu-22.04] openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -231,7 +232,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux @@ -410,7 +411,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 8f56220feea90b..754f179f105591 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -157,7 +157,7 @@ jobs: CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ CPP="$CC --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 6f599f75547ceb..5cd6fb39f1e12f 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -1,6 +1,4 @@ ---- - -name: Change detection +name: Reusable change detection on: # yamllint disable-line rule:truthy workflow_call: diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 4b021b3dc32f15..3809f24dcc977e 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -1,4 +1,4 @@ -name: Docs +name: Reusable Docs on: workflow_call: @@ -95,7 +95,7 @@ jobs: # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index b4227545887ad1..b3a160fbbf8053 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -1,3 +1,5 @@ +name: Reusable macOS + on: workflow_call: inputs: diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 27f4eacd86fd95..f4c976ca996410 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -1,3 +1,5 @@ +name: Reusable Thread Sanitizer + on: workflow_call: inputs: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 01bd914af79fa0..0cf40ba8a9b03b 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -1,3 +1,5 @@ +name: Reusable Ubuntu + on: workflow_call: inputs: @@ -14,7 +16,11 @@ jobs: build_ubuntu_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04] env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 @@ -36,7 +42,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 1b1a68c0badc76..4c8137c958a312 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -1,3 +1,5 @@ +name: Reusable WASI + on: workflow_call: inputs: diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index fc34ab7c3eb1f2..abdb1a1982fef8 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -1,4 +1,4 @@ -name: TestsMSI +name: Reusable Windows MSI on: workflow_call: diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index e9c3c8e05a801c..dcfc62d7f5d145 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -1,3 +1,5 @@ +name: Reusable Windows + on: workflow_call: inputs: diff --git a/Doc/Makefile b/Doc/Makefile index a2d89343648dc1..a090ee5ba92705 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -305,13 +305,15 @@ serve: # for development releases: always build .PHONY: autobuild-dev +autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev: - $(MAKE) dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) -# for quick rebuilds (HTML only) +# for HTML-only rebuilds .PHONY: autobuild-dev-html +autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev-html: - $(MAKE) html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6ae6bfe4aa6ab4..6eae24b38fae48 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -32,11 +32,13 @@ bound into a function. .. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in a code object. + Return the number of :term:`free (closure) variables ` + in a code object. .. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co) - Return the position of the first free variable in a code object. + Return the position of the first :term:`free (closure) variable ` + in a code object. .. versionchanged:: 3.13 @@ -144,7 +146,8 @@ bound into a function. Equivalent to the Python code ``getattr(co, 'co_freevars')``. Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the free variables. On error, ``NULL`` is returned and an exception is raised. + the :term:`free (closure) variables `. On error, ``NULL`` is returned + and an exception is raised. .. versionadded:: 3.11 diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 0de135b232aaaf..8eba54a80dc80d 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -122,18 +122,24 @@ Context object management functions: .. c:type:: PyContextEvent Enumeration of possible context object watcher events: - - ``Py_CONTEXT_EVENT_ENTER`` - - ``Py_CONTEXT_EVENT_EXIT`` + + - ``Py_CONTEXT_EVENT_ENTER``: A context has been entered, causing the + :term:`current context` to switch to it. The object passed to the watch + callback is the now-current :class:`contextvars.Context` object. Each + enter event will eventually have a corresponding exit event for the same + context object after any subsequently entered contexts have themselves been + exited. + - ``Py_CONTEXT_EVENT_EXIT``: A context is about to be exited, which will + cause the :term:`current context` to switch back to what it was before the + context was entered. The object passed to the watch callback is the + still-current :class:`contextvars.Context` object. .. versionadded:: 3.14 -.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyContext* ctx) +.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyObject *obj) - Type of a context object watcher callback function. - If *event* is ``Py_CONTEXT_EVENT_ENTER``, then the callback is invoked - after *ctx* has been set as the current context for the current thread. - Otherwise, the callback is invoked before the deactivation of *ctx* as the current context - and the restoration of the previous contex object for the current thread. + Context object watcher callback function. The object passed to the callback + is event-specific; see :c:type:`PyContextEvent` for details. If the callback returns with an exception set, it must return ``-1``; this exception will be printed as an unraisable exception using diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 05349590975160..fc2336d120c259 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -733,7 +733,7 @@ Exception Classes This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). - The :attr:`!__module__` attribute of the new class is set to the first part (up + The :attr:`~type.__module__` attribute of the new class is set to the first part (up to the last dot) of the *name* argument, and the class name is set to the last part (after the last dot). The *base* argument can be used to specify alternate base classes; it can either be only one class or a tuple of classes. The *dict* diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8108a5015be972..6e48644c8fef8b 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -136,14 +136,14 @@ Importing Modules such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`~importlib.machinery.SourceFileLoader` otherwise. + The module's :attr:`~module.__spec__` and :attr:`~module.__loader__` will be + set, if not set already, with the appropriate values. The spec's loader + will be set to the module's :attr:`!__loader__` (if set) and to an instance + of :class:`~importlib.machinery.SourceFileLoader` otherwise. - The module's :attr:`__file__` attribute will be set to the code object's - :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also - be set. + The module's :attr:`~module.__file__` attribute will be set to the code + object's :attr:`~codeobject.co_filename`. If applicable, + :attr:`~module.__cached__` will also be set. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -155,29 +155,29 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`__cached__` and :attr:`__loader__` is - deprecated. See :class:`~importlib.machinery.ModuleSpec` for + The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) - Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of - the module object is set to *pathname* if it is non-``NULL``. + Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`~module.__file__` + attribute of the module object is set to *pathname* if it is non-``NULL``. See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` + Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` attribute of the module object is set to *cpathname* if it is non-``NULL``. Of the three functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`__cached__` is deprecated. See + Setting :attr:`~module.__cached__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 561c8a1b879bae..8e0cf7bb0fc088 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,7 +7,8 @@ Initialization, Finalization, and Threads ***************************************** -See also the :ref:`Python Initialization Configuration `. +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. .. _pre-init-safe: @@ -21,6 +22,15 @@ a few functions and the :ref:`global configuration variables The following functions can be safely called before Python is initialized: +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + * Configuration functions: * :c:func:`PyImport_AppendInittab` @@ -32,6 +42,7 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` * :c:func:`PySys_ResetWarnOptions` + * the configuration functions covered in :ref:`init-config` * Informative functions: @@ -43,10 +54,12 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_GetCopyright` * :c:func:`Py_GetPlatform` * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` * Utilities: * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` * Memory allocators: @@ -62,11 +75,13 @@ The following functions can be safely called before Python is initialized: .. note:: - The following functions **should not be called** before - :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`. + :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. .. _global-conf-vars: @@ -346,34 +361,42 @@ Initializing and finalizing the interpreter this should be called before using any other Python/C API functions; see :ref:`Before Python Initialization ` for the few exceptions. - This initializes - the table of loaded modules (``sys.modules``), and creates the fundamental - modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes - the module search path (``sys.path``). It does not set ``sys.argv``; use - the new :c:type:`PyConfig` API of the :ref:`Python Initialization - Configuration ` for that. This is a no-op when called for a - second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a - fatal error if the initialization fails. - - Use the :c:func:`Py_InitializeFromConfig` function to customize the + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration `. .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, which will - also affect non-Python uses of the console using the C Runtime. + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. .. c:function:: void Py_InitializeEx(int initsigs) This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, which - might be useful when Python is embedded. + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. - Use the :c:func:`Py_InitializeFromConfig` function to customize the + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration `. +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false @@ -430,18 +453,121 @@ Initializing and finalizing the interpreter Some memory allocated by extension modules may not be freed. Some extensions may not work properly if their initialization routine is called more than once; this can happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` - more than once. + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx .. versionadded:: 3.6 + .. c:function:: void Py_Finalize() This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that disregards the return value. +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value will be ``0`` if the interpreter exits normally (i.e., + without an exception), ``1`` if the interpreter exits due to an exception, + or ``2`` if the argument list does not represent a valid Python command + line. + + Note that if an otherwise unhandled :exc:`SystemExit` is raised, this + function will not return ``1``, but exit the process, as long as + ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will + drop into the interactive Python prompt, at which point a second otherwise + unhandled :exc:`SystemExit` will still exit the process, while any other + means of exiting will set the return value as described above. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), or ``1`` if the interpreter exits due to an exception. If an + otherwise unhandled :exc:`SystemExit` is raised, the function will immediately + exit the process instead of returning ``1``. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: returning ``0`` + if the session terminates without raising an unhandled exception, exiting + immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for + any other unhandled exception. + + This function always finalizes the Python interpreter regardless of whether + it returns a value or immediately exits the process due to an unhandled + :exc:`SystemExit` exception. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + + Process-wide parameters ======================= @@ -960,6 +1086,37 @@ thread, where the CPython global runtime was originally initialized. The only exception is if :c:func:`exec` will be called immediately after. +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`_Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to acquire the GIL +during finalization, either explicitly via :c:func:`PyGILState_Ensure`, +:c:macro:`Py_END_ALLOW_THREADS`, :c:func:`PyEval_AcquireThread`, or +:c:func:`PyEval_AcquireLock`, or implicitly when the interpreter attempts to +reacquire it after having yielded it, the thread enters **a permanently blocked +state** where it remains until the program exits. In most cases this is +harmless, but this can result in deadlock if a later stage of finalization +attempts to acquire a lock owned by the blocked thread, or otherwise waits on +the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime GIL acquiring C APIs +have never had any error reporting or handling expectations at GIL acquisition +time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + High-level API -------------- @@ -1033,11 +1190,14 @@ code, or when embedding the Python interpreter: ensues. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: next + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: PyThreadState* PyThreadState_Get() @@ -1092,11 +1252,14 @@ with sub-interpreters: to call arbitrary Python code. Failure is a fatal error. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: next + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: void PyGILState_Release(PyGILState_STATE) @@ -1224,7 +1387,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyThreadState_DeleteCurrent(void) Destroy the current thread state and release the global interpreter lock. - Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not + Like :c:func:`PyThreadState_Delete`, the global interpreter lock must be held. The thread state must have been reset with a previous call to :c:func:`PyThreadState_Clear`. @@ -1374,17 +1537,20 @@ All of the following functions must be called after :c:func:`Py_Initialize`. If this thread already has the lock, deadlock ensues. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.8 Updated to be consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, and terminate the current thread if called while the interpreter is finalizing. + .. versionchanged:: next + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + :c:func:`PyEval_RestoreThread` is a higher-level function which is always available (even when threads have not been initialized). diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0ef7d015be9b93..6194d7446c73e4 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1248,19 +1248,24 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``1``, enable support for the Linux ``perf`` profiler. - Set by :option:`-X perf <-X>` command-line option and by the - :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support - with stack pointers and :option:`-X perf_jit <-X>` command-line option - and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf - support with DWARF JIT information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. + + Set to ``1`` by :option:`-X perf <-X>` command-line option and the + :envvar:`PYTHONPERFSUPPORT` environment variable. + + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 .. c:member:: int use_environment @@ -1351,14 +1356,13 @@ the :option:`-X` command line option. The ``show_alloc_count`` field has been removed. +.. _init-from-config: + Initialization with PyConfig ---------------------------- -Function to initialize Python: - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration. +Initializing the interpreter from a populated configuration struct is handled +by calling :c:func:`Py_InitializeFromConfig`. The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -1617,6 +1621,8 @@ Create Config Free memory of the initialization configuration *config*. + If *config* is ``NULL``, no operation is performed. + Error Handling -------------- @@ -1819,37 +1825,21 @@ return ``-1`` on error: PyInitConfig_Free(config); return 0; - // Display the error message - const char *err_msg; error: - (void)PyInitConfig_GetError(config, &err_msg); - printf("PYTHON INIT ERROR: %s\n", err_msg); - PyInitConfig_Free(config); + { + // Display the error message + // This uncommon braces style is used, because you cannot make + // goto targets point to variable declarations. + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("PYTHON INIT ERROR: %s\n", err_msg); + PyInitConfig_Free(config); - return -1; + return -1; + } } -Py_RunMain() -============ - -.. c:function:: int Py_RunMain(void) - - Execute the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. - - By default and when if :option:`-i` option is used, run the REPL. - - Finally, finalizes Python and returns an exit status that can be passed to - the ``exit()`` function. - -See :ref:`Python Configuration ` for an example of -customized Python always running in isolated mode using -:c:func:`Py_RunMain`. - - Runtime Python configuration API ================================ diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 098a55c50e219a..02ef8aa7846468 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -159,7 +159,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 -.. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -181,6 +180,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -570,7 +579,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. On failure, return -1 with an exception set. This function always succeeds if *obj* is a :c:type:`PyLongObject` or its subtype. - .. versionadded:: 3.14 + .. versionadded:: next .. c:function:: PyObject* PyLong_GetInfo(void) @@ -599,6 +608,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Exactly what values are considered compact is an implementation detail and is subject to change. + .. versionadded:: 3.12 + + .. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`, @@ -606,3 +618,5 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, the return value is undefined. + .. versionadded:: 3.12 + diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index ec61be284caad9..f82a050ab75de0 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -37,18 +37,19 @@ Module Objects single: __package__ (module attribute) single: __loader__ (module attribute) - Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. + Return a new module object with :attr:`module.__name__` set to *name*. + The module's :attr:`!__name__`, :attr:`~module.__doc__`, + :attr:`~module.__package__` and :attr:`~module.__loader__` attributes are + filled in (all but :attr:`!__name__` are set to ``None``). The caller is + responsible for setting a :attr:`~module.__file__` attribute. Return ``NULL`` with an exception set on error. .. versionadded:: 3.3 .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. + :attr:`~module.__package__` and :attr:`~module.__loader__` are now set to + ``None``. .. c:function:: PyObject* PyModule_New(const char *name) @@ -77,8 +78,9 @@ Module Objects single: __name__ (module attribute) single: SystemError (built-in exception) - Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. + Return *module*'s :attr:`~module.__name__` value. If the module does not + provide one, or if it is not a string, :exc:`SystemError` is raised and + ``NULL`` is returned. .. versionadded:: 3.3 @@ -108,8 +110,8 @@ Module Objects single: SystemError (built-in exception) Return the name of the file from which *module* was loaded using *module*'s - :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return + :attr:`~module.__file__` attribute. If this is not defined, or if it is not a + string, raise :exc:`SystemError` and return ``NULL``; otherwise return a reference to a Unicode object. .. versionadded:: 3.2 diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 1c28f30321bd7a..630114a4339110 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -367,14 +367,14 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to + If *cls* has a :meth:`~type.__subclasscheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. + i.e. contained in :attr:`cls.__mro__ `. Normally only class objects, i.e. instances of :class:`type` or a derived class, are considered classes. However, objects can override this by having - a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). + a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). .. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) @@ -386,15 +386,15 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to + If *cls* has a :meth:`~type.__instancecheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *inst* is an instance of *cls* if its class is a subclass of *cls*. An instance *inst* can override what is considered its class by having a - :attr:`~instance.__class__` attribute. + :attr:`~object.__class__` attribute. An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple + classes are, by having a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 7a8a6134282ade..815afddad19df1 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -167,7 +167,8 @@ type. .. c:member:: const char *name - Name of the struct sequence type. + Fully qualified name of the type; null-terminated UTF-8 encoded. + The name must contain the module name. .. c:member:: const char *doc diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index b56da6954f41d4..0031708c4680cc 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -53,7 +53,8 @@ Type Objects .. c:function:: PyObject* PyType_GetDict(PyTypeObject* type) Return the type object's internal namespace, which is otherwise only - exposed via a read-only proxy (``cls.__dict__``). This is a + exposed via a read-only proxy (:attr:`cls.__dict__ `). + This is a replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly. The returned dictionary must be treated as read-only. @@ -140,7 +141,7 @@ Type Objects Return true if *a* is a subtype of *b*. This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call + :meth:`~type.__subclasscheck__` is not called on *b*. Call :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` would do. @@ -174,29 +175,30 @@ Type Objects .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) - Return the type's name. Equivalent to getting the type's ``__name__`` attribute. + Return the type's name. Equivalent to getting the type's + :attr:`~type.__name__` attribute. .. versionadded:: 3.11 .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the - type's ``__qualname__`` attribute. + type's :attr:`~type.__qualname__` attribute. .. versionadded:: 3.11 .. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type) Return the type's fully qualified name. Equivalent to - ``f"{type.__module__}.{type.__qualname__}"``, or ``type.__qualname__`` if - ``type.__module__`` is not a string or is equal to ``"builtins"``. + ``f"{type.__module__}.{type.__qualname__}"``, or :attr:`type.__qualname__` + if :attr:`type.__module__` is not a string or is equal to ``"builtins"``. .. versionadded:: 3.13 .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) - Return the type's module name. Equivalent to getting the ``type.__module__`` - attribute. + Return the type's module name. Equivalent to getting the + :attr:`type.__module__` attribute. .. versionadded:: 3.13 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index cfe4563d744b8a..da1b5092fbf787 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -567,12 +567,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) For :ref:`statically allocated type objects `, the *tp_name* field should contain a dot. - Everything before the last dot is made accessible as the :attr:`__module__` + Everything before the last dot is made accessible as the :attr:`~type.__module__` attribute, and everything after the last dot is made accessible as the - :attr:`~definition.__name__` attribute. + :attr:`~type.__name__` attribute. If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the - :attr:`~definition.__name__` attribute, and the :attr:`__module__` attribute is undefined + :attr:`~type.__name__` attribute, and the :attr:`~type.__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. Additionally, it will not be listed in module documentations created with pydoc. @@ -1131,7 +1131,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a ``__dict__`` + This bit indicates that instances of the class have a `~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. @@ -1335,8 +1335,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`__doc__` attribute on the type and - instances of the type. + type object. This is exposed as the :attr:`~type.__doc__` attribute on the + type and instances of the type. **Inheritance:** @@ -2036,7 +2036,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) A collection of subclasses. Internal use only. May be an invalid pointer. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__`. + :py:meth:`~type.__subclasses__`. .. versionchanged:: 3.12 diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 958fafd47ac81b..4daf9e9fdbf2f1 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -317,7 +317,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -338,6 +338,8 @@ APIs: This is the recommended way to allocate a new Unicode object. Objects created using this function are not resizable. + On error, set an exception and return ``NULL``. + .. versionadded:: 3.3 @@ -614,6 +616,8 @@ APIs: Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -657,6 +661,8 @@ APIs: not out of bounds, and that the object can be modified safely (i.e. that it its reference count is one). + Return ``0`` on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -666,6 +672,8 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -674,6 +682,7 @@ APIs: Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 @@ -990,6 +999,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -1002,6 +1014,9 @@ These are the UTF-8 codec APIs: On error, set an exception, set *size* to ``-1`` (if it's not NULL) and return ``NULL``. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not responsible for deallocating the buffer. The buffer is deallocated and @@ -1423,14 +1438,40 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function returns ``-1`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. seealso:: + + The :c:func:`PyUnicode_Equal` function. + + +.. c:function:: int PyUnicode_Equal(PyObject *a, PyObject *b) + + Test if two strings are equal: + + * Return ``1`` if *a* is equal to *b*. + * Return ``0`` if *a* is not equal to *b*. + * Set a :exc:`TypeError` exception and return ``-1`` if *a* or *b* is not a + :class:`str` object. + + The function always succeeds if *a* and *b* are :class:`str` objects. + + The function works for :class:`str` subclasses, but does not honor custom + ``__eq__()`` method. + + .. seealso:: + + The :c:func:`PyUnicode_Compare` function. + + .. versionadded:: 3.14 + .. c:function:: int PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *string, Py_ssize_t size) Compare a Unicode object with a char buffer which is interpreted as being UTF-8 or ASCII encoded and return true (``1``) if they are equal, or false (``0``) otherwise. - If the Unicode object contains surrogate characters or - the C string is not valid UTF-8, false (``0``) is returned. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. This function does not raise exceptions. @@ -1534,7 +1575,7 @@ PyUnicodeWriter The :c:type:`PyUnicodeWriter` API can be used to create a Python :class:`str` object. -.. versionadded:: 3.14 +.. versionadded:: next .. c:type:: PyUnicodeWriter @@ -1559,6 +1600,8 @@ object. Discard the internal Unicode buffer and destroy the writer instance. + If *writer* is ``NULL``, no operation is performed. + .. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) Write the single Unicode character *ch* into *writer*. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 67167444d0a685..9f02bdb5896563 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -25,30 +25,6 @@ are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter. This is made available for - programs which embed Python. The *argc* and *argv* parameters should be - prepared exactly as those which are passed to a C program's :c:func:`main` - function (converted to wchar_t according to the user's locale). It is - important to note that the argument list may be modified (but the contents of - the strings pointed to by the argument list are not). The return value will - be ``0`` if the interpreter exits normally (i.e., without an exception), - ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter - list does not represent a valid Python command line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - :c:member:`PyConfig.inspect` is zero. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. - - .. versionadded:: 3.8 - - .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving diff --git a/Doc/conf.py b/Doc/conf.py index 27cf03d6bea05a..839beaad08bebd 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -11,6 +11,8 @@ import sys import time +import sphinx + sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) @@ -21,6 +23,7 @@ extensions = [ 'audit_events', + 'availability', 'c_annotations', 'glossary_search', 'lexers', @@ -61,7 +64,10 @@ # General substitutions. project = 'Python' -copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" +if sphinx.version_info[:2] >= (8, 1): + copyright = "2001-%Y, Python Software Foundation" +else: + copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. @@ -360,10 +366,14 @@ } # This 'Last updated on:' timestamp is inserted at the bottom of every page. -html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) -html_last_updated_fmt = time.strftime( - '%b %d, %Y (%H:%M UTC)', time.gmtime(html_time) -) +html_last_updated_fmt = '%b %d, %Y (%H:%M UTC)' +if sphinx.version_info[:2] >= (8, 1): + html_last_updated_use_utc = True +else: + html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) + html_last_updated_fmt = time.strftime( + html_last_updated_fmt, time.gmtime(html_time) + ) # Path to find HTML templates. templates_path = ['tools/templates'] @@ -413,8 +423,8 @@ \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', - # The paper size ('letter' or 'a4'). - 'papersize': 'a4', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', } @@ -595,13 +605,21 @@ # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { - "cve": ("https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-%s", "CVE-%s"), - "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), "source": (SOURCE_URI, "%s"), } extlinks_detect_hardcoded_links = True +if sphinx.version_info[:2] < (8, 1): + # Sphinx 8.1 has in-built CVE and CWE roles. + extlinks |= { + "cve": ( + "https://www.cve.org/CVERecord?id=CVE-%s", + "CVE-%s", + ), + "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"), + } + # Options for c_annotations # ------------------------- diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 19dc71a345b474..9314facd2ad873 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -783,6 +783,7 @@ func,PyUnicode_DecodeUnicodeEscape,3.2,, func,PyUnicode_EncodeCodePage,3.7,on Windows, func,PyUnicode_EncodeFSDefault,3.2,, func,PyUnicode_EncodeLocale,3.7,, +func,PyUnicode_Equal,3.14,, func,PyUnicode_EqualToUTF8,3.13,, func,PyUnicode_EqualToUTF8AndSize,3.13,, func,PyUnicode_FSConverter,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.14.rst b/Doc/deprecations/c-api-pending-removal-in-3.14.rst index d16da66c29abe7..9e10bf2691e5c8 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.14.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.14.rst @@ -1,4 +1,4 @@ -Pending Removal in Python 3.14 +Pending removal in Python 3.14 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index e3974415e0cc89..1bb49e5b4874f2 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -1,4 +1,4 @@ -Pending Removal in Python 3.15 +Pending removal in Python 3.15 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * The bundled copy of ``libmpdecimal``. diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst index 0c3ae52b87ff74..8fc1c80c35d092 100644 --- a/Doc/deprecations/c-api-pending-removal-in-future.rst +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -1,4 +1,4 @@ -Pending Removal in Future Versions +Pending removal in future versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The following APIs are deprecated and will be removed, diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index a9efb0bc744335..bac6e3f18d4594 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -7,7 +7,7 @@ Deprecations .. include:: pending-removal-in-future.rst -C API Deprecations +C API deprecations ------------------ .. include:: c-api-pending-removal-in-3.15.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 89790497816e83..2fd2f12cc6a2c4 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -1,4 +1,4 @@ -Pending Removal in Python 3.13 +Pending removal in Python 3.13 ------------------------------ Modules (see :pep:`594`): diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 452d6643e1d146..b8791b8d6c387e 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -1,6 +1,13 @@ -Pending Removal in Python 3.14 +Pending removal in Python 3.14 ------------------------------ +* The import system: + + * Setting :attr:`~module.__loader__` on a module while + failing to set :attr:`__spec__.loader ` + is deprecated. In Python 3.14, :attr:`!__loader__` will cease to be set or + taken into consideration by the import system or the standard library. + * :mod:`argparse`: The *type*, *choices*, and *metavar* parameters of :class:`!argparse.BooleanOptionalAction` are deprecated and will be removed in 3.14. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index b921b4f97d524e..17029b8d4773bd 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -1,6 +1,18 @@ -Pending Removal in Python 3.15 +Pending removal in Python 3.15 ------------------------------ +* The import system: + + * Setting :attr:`~module.__cached__` on a module while + failing to set :attr:`__spec__.cached ` + is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + * :mod:`ctypes`: * The undocumented :func:`!ctypes.SetPointerType` function @@ -17,9 +29,6 @@ Pending Removal in Python 3.15 * The :option:`!--cgi` flag to the :program:`python -m http.server` command-line interface has been deprecated since Python 3.13. -* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or - taken into consideration by the import system (:gh:`97879`). - * :class:`locale`: * The :func:`~locale.getdefaultlocale` function @@ -54,7 +63,7 @@ Pending Removal in Python 3.15 * The undocumented keyword argument syntax for creating :class:`~typing.NamedTuple` classes - (e.g. ``Point = NamedTuple("Point", x=int, y=int)``) + (for example, ``Point = NamedTuple("Point", x=int, y=int)``) has been deprecated since Python 3.13. Use the class-based syntax or the functional syntax instead. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 446cc63cb34ff9..fac500d34742ca 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -1,6 +1,21 @@ -Pending Removal in Python 3.16 +Pending removal in Python 3.16 ------------------------------ +* :mod:`array`: + + * The ``'u'`` format code (:c:type:`wchar_t`) + has been deprecated in documentation since Python 3.3 + and at runtime since Python 3.13. + Use the ``'w'`` format code (:c:type:`Py_UCS4`) + for Unicode characters instead. + +* :mod:`asyncio`: + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`builtins`: * Bitwise inversion on boolean types, ``~True`` or ``~False`` @@ -10,14 +25,6 @@ Pending Removal in Python 3.16 In the rare case that you need the bitwise inversion of the underlying integer, convert to ``int`` explicitly (``~int(x)``). -* :mod:`array`: - - * The ``'u'`` format code (:c:type:`wchar_t`) - has been deprecated in documentation since Python 3.3 - and at runtime since Python 3.13. - Use the ``'w'`` format code (:c:type:`Py_UCS4`) - for Unicode characters instead. - * :mod:`shutil`: * The :class:`!ExecError` exception diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 3f9cf6f208221a..f916797c07a068 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -1,4 +1,4 @@ -Pending Removal in Future Versions +Pending removal in future versions ---------------------------------- The following APIs will be removed in the future, diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index fd05c82b41629a..7f57a3a6aac0ed 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -296,7 +296,7 @@ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`!__doc__` attribute. +class object, and get the doc string using its :attr:`~type.__doc__` attribute. As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index b8f437f8d2646e..bcf938f117d148 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -144,7 +144,7 @@ only used for variable-sized objects and should otherwise be zero. If you want your type to be subclassable from Python, and your type has the same :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's + in its :attr:`~type.__bases__`, or else it will not be able to call your type's :meth:`~object.__new__` method without getting an error. You can avoid this problem by ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 8f9b464ccbfcb7..fa7b22bde1dc6f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:meth:`self.__dict__ ` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ ` directly. How do I call a method defined in a base class from a derived class that extends it? diff --git a/Doc/glossary.rst b/Doc/glossary.rst index b3fd3c96b5c217..f67f3ecad0bc40 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -231,6 +231,28 @@ Glossary A variable defined in a class and intended to be modified only at class level (i.e., not in an instance of the class). + closure variable + A :term:`free variable` referenced from a :term:`nested scope` that is defined in an outer + scope rather than being resolved at runtime from the globals or builtin namespaces. + May be explicitly defined with the :keyword:`nonlocal` keyword to allow write access, + or implicitly defined if the variable is only being read. + + For example, in the ``inner`` function in the following code, both ``x`` and ``print`` are + :term:`free variables `, but only ``x`` is a *closure variable*:: + + def outer(): + x = 0 + def inner(): + nonlocal x + x += 1 + print(x) + return inner + + Due to the :attr:`codeobject.co_freevars` attribute (which, despite its name, only + includes the names of closure variables rather than listing all referenced free + variables), the more general :term:`free variable` term is sometimes used even + when the intended meaning is to refer specifically to closure variables. + complex number An extension of the familiar real number system in which all numbers are expressed as a sum of a real part and an imaginary part. Imaginary @@ -243,19 +265,33 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + context + This term has different meanings depending on where and how it is used. + Some common meanings: + + * The temporary state or environment established by a :term:`context + manager` via a :keyword:`with` statement. + * The collection of key­value bindings associated with a particular + :class:`contextvars.Context` object and accessed via + :class:`~contextvars.ContextVar` objects. Also see :term:`context + variable`. + * A :class:`contextvars.Context` object. Also see :term:`current + context`. + + context management protocol + The :meth:`~object.__enter__` and :meth:`~object.__exit__` methods called + by the :keyword:`with` statement. See :pep:`343`. + context manager - An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. - See :pep:`343`. + An object which implements the :term:`context management protocol` and + controls the environment seen in a :keyword:`with` statement. See + :pep:`343`. context variable - A variable which can have different values depending on its context. - This is similar to Thread-Local Storage in which each execution - thread may have a different value for a variable. However, with context - variables, there may be several contexts in one execution thread and the - main usage for context variables is to keep track of variables in + A variable whose value depends on which context is the :term:`current + context`. Values are accessed via :class:`contextvars.ContextVar` + objects. Context variables are primarily used to isolate state between concurrent asynchronous tasks. - See :mod:`contextvars`. contiguous .. index:: C-contiguous, Fortran contiguous @@ -289,6 +325,14 @@ Glossary is used when necessary to distinguish this implementation from others such as Jython or IronPython. + current context + The :term:`context` (:class:`contextvars.Context` object) that is + currently used by :class:`~contextvars.ContextVar` objects to access (get + or set) the values of :term:`context variables `. Each + thread has its own current context. Frameworks for executing asynchronous + tasks (see :mod:`asyncio`) associate each task with a context which + becomes the current context whenever the task starts or resumes execution. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -347,7 +391,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`!__doc__` attribute + recognized by the compiler and put into the :attr:`~definition.__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -439,7 +483,7 @@ Glossary ` for use with :data:`sys.meta_path`, and :term:`path entry finders ` for use with :data:`sys.path_hooks`. - See :ref:`importsystem` and :mod:`importlib` for much more detail. + See :ref:`finders-and-loaders` and :mod:`importlib` for much more detail. floor division Mathematical division that rounds down to nearest integer. The floor @@ -454,6 +498,13 @@ Glossary the :term:`global interpreter lock` which allows only one thread to execute Python bytecode at a time. See :pep:`703`. + free variable + Formally, as defined in the :ref:`language execution model `, a free + variable is any variable used in a namespace which is not a local variable in that + namespace. See :term:`closure variable` for an example. + Pragmatically, due to the name of the :attr:`codeobject.co_freevars` attribute, + the term is also sometimes used as a synonym for :term:`closure variable`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments ` which may be used in @@ -566,7 +617,7 @@ Glossary As of Python 3.13, the GIL can be disabled using the :option:`--disable-gil` build configuration. After building Python with this option, code must be - run with :option:`-X gil 0 <-X>` or after setting the :envvar:`PYTHON_GIL=0 ` + run with :option:`-X gil=0 <-X>` or after setting the :envvar:`PYTHON_GIL=0 ` environment variable. This feature enables improved performance for multi-threaded applications and makes it easier to use multi-core CPUs efficiently. For more details, see :pep:`703`. @@ -762,8 +813,11 @@ Glossary loader An object that loads a module. It must define a method named :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See :pep:`302` for details and - :class:`importlib.abc.Loader` for an :term:`abstract base class`. + :term:`finder`. See also: + + * :ref:`finders-and-loaders` + * :class:`importlib.abc.Loader` + * :pep:`302` locale encoding On Unix, it is the encoding of the LC_CTYPE locale. It can be set with @@ -833,6 +887,8 @@ Glossary A namespace containing the import-related information used to load a module. An instance of :class:`importlib.machinery.ModuleSpec`. + See also :ref:`module-specs`. + MRO See :term:`method resolution order`. @@ -1160,16 +1216,12 @@ Glossary (subscript) notation uses :class:`slice` objects internally. soft deprecated - A soft deprecation can be used when using an API which should no longer - be used to write new code, but it remains safe to continue using it in - existing code. The API remains documented and tested, but will not be - developed further (no enhancement). - - The main difference between a "soft" and a (regular) "hard" deprecation - is that the soft deprecation does not imply scheduling the removal of the - deprecated API. + A soft deprecated API should not be used in new code, + but it is safe for already existing code to use it. + The API remains documented and tested, but will not be enhanced further. - Another difference is that a soft deprecation does not issue a warning. + Soft deprecation, unlike normal deprecation, does not plan on removing the API + and will not emit warnings. See `PEP 387: Soft Deprecation `_. @@ -1241,7 +1293,7 @@ Glossary type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with + :attr:`~object.__class__` attribute or can be retrieved with ``type(obj)``. type alias diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index e9fc563f1b5880..78f3704ba5d000 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -107,9 +107,9 @@ Your code will have to have a separate code path if the object you're examining is a class (``isinstance(o, type)``). In that case, best practice relies on an implementation detail of Python 3.9 and before: if a class has annotations defined, -they are stored in the class's ``__dict__`` dictionary. Since +they are stored in the class's :attr:`~type.__dict__` dictionary. Since the class may or may not have annotations defined, best practice -is to call the ``get`` method on the class dict. +is to call the :meth:`~dict.get` method on the class dict. To put it all together, here is some sample code that safely accesses the ``__annotations__`` attribute on an arbitrary @@ -126,8 +126,8 @@ the type of ``ann`` using :func:`isinstance` before further examination. Note that some exotic or malformed type objects may not have -a ``__dict__`` attribute, so for extra safety you may also wish -to use :func:`getattr` to access ``__dict__``. +a :attr:`~type.__dict__` attribute, so for extra safety you may also wish +to use :func:`getattr` to access :attr:`!__dict__`. Manually Un-Stringizing Stringized Annotations @@ -247,4 +247,5 @@ on the class, you may observe unexpected behavior; see quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or :func:`inspect.get_annotations` on Python 3.10+. On earlier versions of Python, you can avoid these bugs by accessing the annotations from the -class's ``__dict__`` (e.g., ``cls.__dict__.get('__annotations__', None)``). +class's :attr:`~type.__dict__` +(e.g., ``cls.__dict__.get('__annotations__', None)``). diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst new file mode 100644 index 00000000000000..cef2d893b28a62 --- /dev/null +++ b/Doc/howto/argparse-optparse.rst @@ -0,0 +1,55 @@ +.. currentmodule:: argparse + +.. _upgrading-optparse-code: + +========================== +Upgrading optparse code +========================== + +Originally, the :mod:`argparse` module had attempted to maintain compatibility +with :mod:`optparse`. However, :mod:`optparse` was difficult to extend +transparently, particularly with the changes required to support +``nargs=`` specifiers and better usage messages. When most everything in +:mod:`optparse` had either been copy-pasted over or monkey-patched, it no +longer seemed practical to try to maintain the backwards compatibility. + +The :mod:`argparse` module improves on the :mod:`optparse` +module in a number of ways including: + +* Handling positional arguments. +* Supporting subcommands. +* Allowing alternative option prefixes like ``+`` and ``/``. +* Handling zero-or-more and one-or-more style arguments. +* Producing more informative usage messages. +* Providing a much simpler interface for custom ``type`` and ``action``. + +A partial upgrade path from :mod:`optparse` to :mod:`argparse`: + +* Replace all :meth:`optparse.OptionParser.add_option` calls with + :meth:`ArgumentParser.add_argument` calls. + +* Replace ``(options, args) = parser.parse_args()`` with ``args = + parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` + calls for the positional arguments. Keep in mind that what was previously + called ``options``, now in the :mod:`argparse` context is called ``args``. + +* Replace :meth:`optparse.OptionParser.disable_interspersed_args` + by using :meth:`~ArgumentParser.parse_intermixed_args` instead of + :meth:`~ArgumentParser.parse_args`. + +* Replace callback actions and the ``callback_*`` keyword arguments with + ``type`` or ``action`` arguments. + +* Replace string names for ``type`` keyword arguments with the corresponding + type objects (e.g. int, float, complex, etc). + +* Replace :class:`optparse.Values` with :class:`Namespace` and + :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with + :exc:`ArgumentError`. + +* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with + the standard Python syntax to use dictionaries to format strings, that is, + ``%(default)s`` and ``%(prog)s``. + +* Replace the OptionParser constructor ``version`` argument with a call to + ``parser.add_argument('--version', action='version', version='')``. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index d1101648f9d8ae..01264bfe823746 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -562,8 +562,8 @@ attribute access. The expression ``obj.x`` looks up the attribute ``x`` in the chain of namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. +instance :attr:`~object.__dict__`, its :meth:`~object.__get__` method is +invoked according to the precedence rules listed below. The details of invocation depend on whether ``obj`` is an object, class, or instance of super. diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index f406873226196b..66929b4104d8de 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -608,7 +608,7 @@ The solution is to specify the module name explicitly as follows:: the source, pickling will be disabled. The new pickle protocol 4 also, in some circumstances, relies on -:attr:`~definition.__qualname__` being set to the location where pickle will be able +:attr:`~type.__qualname__` being set to the location where pickle will be able to find the class. For example, if the class was made available in class SomeData in the global scope:: diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst new file mode 100644 index 00000000000000..b21e3287ecaa3f --- /dev/null +++ b/Doc/howto/free-threading-python.rst @@ -0,0 +1,154 @@ +.. _freethreading-python-howto: + +********************************************** +Python experimental support for free threading +********************************************** + +Starting with the 3.13 release, CPython has experimental support for a build of +Python called :term:`free threading` where the :term:`global interpreter lock` +(GIL) is disabled. Free-threaded execution allows for full utilization of the +available processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. + +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. + +This document describes the implications of free threading +for Python code. See :ref:`freethreading-extensions-howto` for information on +how to write C extensions that support the free-threaded build. + +.. seealso:: + + :pep:`703` – Making the Global Interpreter Lock Optional in CPython for an + overall description of free-threaded Python. + + +Installation +============ + +Starting with Python 3.13, the official macOS and Windows installers +optionally support installing free-threaded Python binaries. The installers +are available at https://www.python.org/downloads/. + +For information on other platforms, see the `Installing a Free-Threaded Python +`_, a +community-maintained installation guide for installing free-threaded Python. + +When building CPython from source, the :option:`--disable-gil` configure option +should be used to build a free-threaded Python interpreter. + + +Identifying free-threaded Python +================================ + +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. + +The ``sysconfig.get_config_var("Py_GIL_DISABLED")`` configuration variable can +be used to determine whether the build supports free threading. If the variable +is set to ``1``, then the build supports free threading. This is the recommended +mechanism for decisions related to the build configuration. + + +The global interpreter lock in free-threaded Python +=================================================== + +Free-threaded builds of CPython support optionally running with the GIL enabled +at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil`. + +The GIL may also automatically be enabled when importing a C-API extension +module that is not explicitly marked as supporting free threading. A warning +will be printed in this case. + +In addition to individual package documentation, the following websites track +the status of popular packages support for free threading: + +* https://py-free-threading.github.io/tracking/ +* https://hugovk.github.io/free-threaded-wheels/ + + +Thread safety +============= + +The free-threaded build of CPython aims to provide similar thread-safety +behavior at the Python level to the default GIL-enabled build. Built-in +types like :class:`dict`, :class:`list`, and :class:`set` use internal locks +to protect against concurrent modifications in ways that behave similarly to +the GIL. However, Python has not historically guaranteed specific behavior for +concurrent modifications to these built-in types, so this should be treated +as a description of the current implementation, not a guarantee of current or +future behavior. + +.. note:: + + It's recommended to use the :class:`threading.Lock` or other synchronization + primitives instead of relying on the internal locks of built-in types, when + possible. + + +Known limitations +================= + +This section describes known limitations of the free-threaded CPython build. + +Immortalization +--------------- + +The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +Immortal objects are not deallocated and have reference counts that are +never modified. This is done to avoid reference count contention that would +prevent efficient multi-threaded scaling. + +An object will be made immortal when a new thread is started for the first time +after the main thread is running. The following objects are immortalized: + +* :ref:`function ` objects declared at the module level +* :ref:`method ` descriptors +* :ref:`code ` objects +* :term:`module` objects and their dictionaries +* :ref:`classes ` (type objects) + +Because immortal objects are never deallocated, applications that create many +objects of these types may see increased memory usage. This is expected to be +addressed in the 3.14 release. + +Additionally, numeric and string literals in the code as well as strings +returned by :func:`sys.intern` are also immortalized. This behavior is +expected to remain in the 3.14 free-threaded build. + + +Frame objects +------------- + +It is not safe to access :ref:`frame ` objects from other +threads and doing so may cause your program to crash . This means that +:func:`sys._current_frames` is generally not safe to use in a free-threaded +build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` +are generally safe as long as the resulting frame object is not passed to +another thread. + +Iterators +--------- + +Sharing the same iterator object between multiple threads is generally not +safe and threads may see duplicate or missing elements when iterating or crash +the interpreter. + + +Single-threaded performance +--------------------------- + +The free-threaded build has additional overhead when executing Python code +compared to the default GIL-enabled build. In 3.13, this overhead is about +40% on the `pyperformance `_ suite. +Programs that spend most of their time in C extensions or I/O will see +less of an impact. The largest impact is because the specializing adaptive +interpreter (:pep:`659`) is disabled in the free-threaded build. We expect +to re-enable it in a thread-safe way in the 3.14 release. This overhead is +expected to be reduced in upcoming Python release. We are aiming for an +overhead of 10% or less on the pyperformance suite compared to the default +GIL-enabled build. diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index a882f1747084fe..c09f92c9528ee1 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -32,6 +32,7 @@ Python Library Reference. isolating-extensions.rst timerfd.rst mro.rst + free-threading-python.rst free-threading-extensions.rst General: @@ -52,6 +53,7 @@ General: Advanced development: * :ref:`curses-howto` +* :ref:`freethreading-python-howto` * :ref:`freethreading-extensions-howto` * :ref:`isolating-extensions-howto` * :ref:`python_2.3_mro` diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index f44b4f98e570bd..46db516e16dae4 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -335,7 +335,7 @@ E is more specialized than C, even if it is in a higher level. A lazy programmer can obtain the MRO directly from Python 2.2, since in this case it coincides with the Python 2.3 linearization. It is enough -to invoke the .mro() method of class A: +to invoke the :meth:`~type.mro` method of class A: >>> A.mro() # doctest: +NORMALIZE_WHITESPACE [, , , diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 0bbf97da39768d..cbc49d15a0771b 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -100,8 +100,8 @@ mainloop of the web server:: (clientsocket, address) = serversocket.accept() # now do something with the clientsocket # in this case, we'll pretend this is a threaded server - ct = client_thread(clientsocket) - ct.run() + ct = make_client_thread(clientsocket) + ct.start() There's actually 3 general ways in which this loop could work - dispatching a thread to handle ``clientsocket``, create a new process to handle diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index b98f91e023bdfc..70c34cde8a0659 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -47,11 +47,14 @@ lists. In contrast, the :func:`sorted` function accepts any iterable. Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making +The :meth:`list.sort` method and the functions :func:`sorted`, +:func:`min`, :func:`max`, :func:`heapq.nsmallest`, and +:func:`heapq.nlargest` have a *key* parameter to specify a function (or +other callable) to be called on each list element prior to making comparisons. -For example, here's a case-insensitive string comparison: +For example, here's a case-insensitive string comparison using +:meth:`str.casefold`: .. doctest:: @@ -272,6 +275,70 @@ to make it usable as a key function:: sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order +Strategies For Unorderable Types and Values +=========================================== + +A number of type and value issues can arise when sorting. +Here are some strategies that can help: + +* Convert non-comparable input types to strings prior to sorting: + +.. doctest:: + + >>> data = ['twelve', '11', 10] + >>> sorted(map(str, data)) + ['10', '11', 'twelve'] + +This is needed because most cross-type comparisons raise a +:exc:`TypeError`. + +* Remove special values prior to sorting: + +.. doctest:: + + >>> from math import isnan + >>> from itertools import filterfalse + >>> data = [3.3, float('nan'), 1.1, 2.2] + >>> sorted(filterfalse(isnan, data)) + [1.1, 2.2, 3.3] + +This is needed because the `IEEE-754 standard +`_ specifies that, "Every NaN +shall compare unordered with everything, including itself." + +Likewise, ``None`` can be stripped from datasets as well: + +.. doctest:: + + >>> data = [3.3, None, 1.1, 2.2] + >>> sorted(x for x in data if x is not None) + [1.1, 2.2, 3.3] + +This is needed because ``None`` is not comparable to other types. + +* Convert mapping types into sorted item lists before sorting: + +.. doctest:: + + >>> data = [{'a': 1}, {'b': 2}] + >>> sorted(data, key=lambda d: sorted(d.items())) + [{'a': 1}, {'b': 2}] + +This is needed because dict-to-dict comparisons raise a +:exc:`TypeError`. + +* Convert set types into sorted lists before sorting: + +.. doctest:: + + >>> data = [{'a', 'b', 'c'}, {'b', 'c', 'd'}] + >>> sorted(map(sorted, data)) + [['a', 'b', 'c'], ['b', 'c', 'd']] + +This is needed because the elements contained in set types do not have a +deterministic order. For example, ``list({'a', 'b'})`` may produce +either ``['a', 'b']`` or ``['b', 'a']``. + Odds and Ends ============= diff --git a/Doc/includes/newtypes/custom2.c b/Doc/includes/newtypes/custom2.c index a0222b1795209b..768ce29fab9ff0 100644 --- a/Doc/includes/newtypes/custom2.c +++ b/Doc/includes/newtypes/custom2.c @@ -23,12 +23,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/newtypes/custom3.c b/Doc/includes/newtypes/custom3.c index 4aeebe0a7507d1..7d969adfa7c9cc 100644 --- a/Doc/includes/newtypes/custom3.c +++ b/Doc/includes/newtypes/custom3.c @@ -23,12 +23,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/newtypes/custom4.c b/Doc/includes/newtypes/custom4.c index 3998918f68301e..a7b8de44a57c90 100644 --- a/Doc/includes/newtypes/custom4.c +++ b/Doc/includes/newtypes/custom4.c @@ -39,12 +39,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 6a1179434acd5a..4f3b663006fb28 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -66,7 +66,7 @@ language using this mechanism: +------------------+-------------+--------------+---------------------------------------------+ | annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | | | | | *Postponed evaluation of annotations*, | -| | | | :pep:`649`: *Deferred evalutation of | +| | | | :pep:`649`: *Deferred evaluation of | | | | | annotations using descriptors* | +------------------+-------------+--------------+---------------------------------------------+ diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 81f0cac947f602..6a66fc4c64bc45 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -213,23 +213,20 @@ In addition to these methods, lock objects can also be used via the .. index:: pair: module; signal -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) +* Interrupts always go to the main thread (the :exc:`KeyboardInterrupt` + exception will be received by that thread.) * Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is equivalent to calling :func:`_thread.exit`. -* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on - a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock - has been acquired. +* It is platform-dependent whether the :meth:`~threading.Lock.acquire` method + on a lock can be interrupted (so that the :exc:`KeyboardInterrupt` exception + will happen immediately, rather than only after the lock has been acquired or + the operation has timed out). It can be interrupted on POSIX, but not on + Windows. * When the main thread exits, it is system defined whether the other threads survive. On most systems, they are killed without executing :keyword:`try` ... :keyword:`finally` clauses or executing object destructors. -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 168ef3ec00d81b..38d744e97d087d 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -99,7 +99,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: that you can customize the behavior of :func:`issubclass` further without the need to call :meth:`register` on every class you want to consider a subclass of the ABC. (This class method is called from the - :meth:`~class.__subclasscheck__` method of the ABC.) + :meth:`~type.__subclasscheck__` method of the ABC.) This method should return ``True``, ``False`` or :data:`NotImplemented`. If it returns ``True``, the *subclass* is considered a subclass of this ABC. @@ -149,7 +149,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: The :meth:`__subclasshook__` class method defined here says that any class that has an :meth:`~iterator.__iter__` method in its :attr:`~object.__dict__` (or in that of one of its base classes, accessed - via the :attr:`~class.__mro__` list) is considered a ``MyIterable`` too. + via the :attr:`~type.__mro__` list) is considered a ``MyIterable`` too. Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 1e72c5421674bc..37490456d13312 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -32,7 +32,7 @@ This module supports retrieving annotations in three main formats for annotations that cannot be resolved, allowing you to inspect the annotations without evaluating them. This is useful when you need to work with annotations that may contain unresolved forward references. -* :attr:`~Format.SOURCE` returns the annotations as a string, similar +* :attr:`~Format.STRING` returns the annotations as a string, similar to how it would appear in the source file. This is useful for documentation generators that want to display annotations in a readable way. @@ -135,7 +135,7 @@ Classes values. Real objects may contain references to, :class:`ForwardRef` proxy objects. - .. attribute:: SOURCE + .. attribute:: STRING :value: 3 Values are the text string of the annotation as it appears in the @@ -197,6 +197,27 @@ Classes Functions --------- +.. function:: annotations_to_string(annotations) + + Convert an annotations dict containing runtime values to a + dict containing only strings. If the values are not already strings, + they are converted using :func:`value_to_string`. + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.STRING` format but + do not have access to the code creating the annotations. + + For example, this is used to implement the :attr:`~Format.STRING` for + :class:`typing.TypedDict` classes created through the functional syntax: + + .. doctest:: + + >>> from typing import TypedDict + >>> Movie = TypedDict("movie", {"name": str, "year": int}) + >>> get_annotations(Movie, format=Format.STRING) + {'name': 'str', 'year': 'int'} + + .. versionadded:: 3.14 + .. function:: call_annotate_function(annotate, format, *, owner=None) Call the :term:`annotate function` *annotate* with the given *format*, @@ -261,7 +282,7 @@ Functions NameError: name 'undefined' is not defined >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) ForwardRef('undefined') - >>> call_evaluate_function(Alias.evaluate_value, Format.SOURCE) + >>> call_evaluate_function(Alias.evaluate_value, Format.STRING) 'undefined' .. versionadded:: 3.14 @@ -347,3 +368,18 @@ Functions {'a': , 'b': , 'return': } .. versionadded:: 3.14 + +.. function:: value_to_string(value) + + Convert an arbitrary Python value to a format suitable for use by the + :attr:`~Format.STRING` format. This calls :func:`repr` for most + objects, but has special handling for some objects, such as type objects. + + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.STRING` format but + do not have access to the code creating the annotations. It can also + be used to provide a user-friendly string representation for other + objects that contain values that are commonly encountered in annotations. + + .. versionadded:: 3.14 + diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 53ecc97d5659f4..d58c75eef3e739 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1,4 +1,4 @@ -:mod:`!argparse` --- Parser for command-line options, arguments and sub-commands +:mod:`!argparse` --- Parser for command-line options, arguments and subcommands ================================================================================ .. module:: argparse @@ -19,17 +19,13 @@ introduction to Python command-line parsing, have a look at the :ref:`argparse tutorial `. -The :mod:`argparse` module makes it easy to write user-friendly command-line -interfaces. The program defines what arguments it requires, and :mod:`argparse` -will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse` +The :mod:`!argparse` module makes it easy to write user-friendly command-line +interfaces. The program defines what arguments it requires, and :mod:`!argparse` +will figure out how to parse those out of :data:`sys.argv`. The :mod:`!argparse` module also automatically generates help and usage messages. The module will also issue errors when users give the program invalid arguments. - -Core Functionality ------------------- - -The :mod:`argparse` module's support for command-line interfaces is built +The :mod:`!argparse` module's support for command-line interfaces is built around an instance of :class:`argparse.ArgumentParser`. It is a container for argument specifications and has options that apply to the parser as whole:: @@ -53,133 +49,9 @@ the extracted data in a :class:`argparse.Namespace` object:: args = parser.parse_args() print(args.filename, args.count, args.verbose) - -Quick Links for add_argument() ------------------------------- - -============================ =========================================================== ========================================================================================================================== -Name Description Values -============================ =========================================================== ========================================================================================================================== -action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` -choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance -const_ Store a constant value -default_ Default value used when an argument is not provided Defaults to ``None`` -dest_ Specify the attribute name used in the result namespace -help_ Help message for an argument -metavar_ Alternate display name for the argument as shown in help -nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` -required_ Indicate whether an argument is required or optional ``True`` or ``False`` -:ref:`type ` Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function -============================ =========================================================== ========================================================================================================================== - - -Example -------- - -The following code is a Python program that takes a list of integers and -produces either the sum or the max:: - - import argparse - - parser = argparse.ArgumentParser(description='Process some integers.') - parser.add_argument('integers', metavar='N', type=int, nargs='+', - help='an integer for the accumulator') - parser.add_argument('--sum', dest='accumulate', action='store_const', - const=sum, default=max, - help='sum the integers (default: find the max)') - - args = parser.parse_args() - print(args.accumulate(args.integers)) - -Assuming the above Python code is saved into a file called ``prog.py``, it can -be run at the command line and it provides useful help messages: - -.. code-block:: shell-session - - $ python prog.py -h - usage: prog.py [-h] [--sum] N [N ...] - - Process some integers. - - positional arguments: - N an integer for the accumulator - - options: - -h, --help show this help message and exit - --sum sum the integers (default: find the max) - -When run with the appropriate arguments, it prints either the sum or the max of -the command-line integers: - -.. code-block:: shell-session - - $ python prog.py 1 2 3 4 - 4 - - $ python prog.py 1 2 3 4 --sum - 10 - -If invalid arguments are passed in, an error will be displayed: - -.. code-block:: shell-session - - $ python prog.py a b c - usage: prog.py [-h] [--sum] N [N ...] - prog.py: error: argument N: invalid int value: 'a' - -The following sections walk you through this example. - - -Creating a parser -^^^^^^^^^^^^^^^^^ - -The first step in using the :mod:`argparse` is creating an -:class:`ArgumentParser` object:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - -The :class:`ArgumentParser` object will hold all the information necessary to -parse the command line into Python data types. - - -Adding arguments -^^^^^^^^^^^^^^^^ - -Filling an :class:`ArgumentParser` with information about program arguments is -done by making calls to the :meth:`~ArgumentParser.add_argument` method. -Generally, these calls tell the :class:`ArgumentParser` how to take the strings -on the command line and turn them into objects. This information is stored and -used when :meth:`~ArgumentParser.parse_args` is called. For example:: - - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.add_argument('--sum', dest='accumulate', action='store_const', - ... const=sum, default=max, - ... help='sum the integers (default: find the max)') - -Later, calling :meth:`~ArgumentParser.parse_args` will return an object with -two attributes, ``integers`` and ``accumulate``. The ``integers`` attribute -will be a list of one or more integers, and the ``accumulate`` attribute will be -either the :func:`sum` function, if ``--sum`` was specified at the command line, -or the :func:`max` function if it was not. - - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` parses arguments through the -:meth:`~ArgumentParser.parse_args` method. This will inspect the command line, -convert each argument to the appropriate type and then invoke the appropriate action. -In most cases, this means a simple :class:`Namespace` object will be built up from -attributes parsed out of the command line:: - - >>> parser.parse_args(['--sum', '7', '-1', '42']) - Namespace(accumulate=, integers=[7, -1, 42]) - -In a script, :meth:`~ArgumentParser.parse_args` will typically be called with no -arguments, and the :class:`ArgumentParser` will automatically determine the -command-line arguments from :data:`sys.argv`. - +.. note:: + If you're looking a guide about how to upgrade optparse code + to argparse, see :ref:`Upgrading Optparse Code `. ArgumentParser objects ---------------------- @@ -195,8 +67,8 @@ ArgumentParser objects as keyword arguments. Each parameter has its own more detailed description below, but in short they are: - * prog_ - The name of the program (default: - ``os.path.basename(sys.argv[0])``) + * prog_ - The name of the program (default: generated from the ``__main__`` + module attributes and ``sys.argv[0]``) * usage_ - The string describing the program usage (default: generated from arguments added to parser) @@ -249,39 +121,21 @@ The following sections describe how each of these are used. prog ^^^^ -By default, :class:`ArgumentParser` objects use the base name -(see :func:`os.path.basename`) of ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match the name that was -used to invoke the program on the command line. For example, consider a file -named ``myprogram.py`` with the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from): - -.. code-block:: shell-session - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] +By default, :class:`ArgumentParser` calculates the name of the program +to display in help messages depending on the way the Python interpreter was run: - options: - -h, --help show this help message and exit - --foo FOO foo help - $ cd .. - $ python subdir/myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help +* The :func:`base name ` of ``sys.argv[0]`` if a file was + passed as argument. +* The Python interpreter name followed by ``sys.argv[0]`` if a directory or + a zipfile was passed as argument. +* The Python interpreter name followed by ``-m`` followed by the + module or package name if the :option:`-m` option was used. -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: +This default is almost always desirable because it will make the help messages +match the string that was used to invoke the program on the command line. +However, to change this default behavior, another value can be supplied using +the ``prog=`` argument to :class:`ArgumentParser`:: >>> parser = argparse.ArgumentParser(prog='myprogram') >>> parser.print_help() @@ -290,7 +144,8 @@ To change this default behavior, another value can be supplied using the options: -h, --help show this help message and exit -Note that the program name, whether determined from ``sys.argv[0]`` or from the +Note that the program name, whether determined from ``sys.argv[0]``, +from the ``__main__`` module attributes or from the ``prog=`` argument, is available to help messages using the ``%(prog)s`` format specifier. @@ -305,27 +160,16 @@ specifier. -h, --help show this help message and exit --foo FOO foo of the myprogram program +.. versionchanged:: 3.14 + The default ``prog`` value now reflects how ``__main__`` was actually executed, + rather than always being ``os.path.basename(sys.argv[0])``. usage ^^^^^ By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help - - options: - -h, --help show this help message and exit - --foo [FOO] foo help - -The default message can be overridden with the ``usage=`` keyword argument:: +arguments it contains. The default message can be overridden with the +``usage=`` keyword argument:: >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') >>> parser.add_argument('--foo', nargs='?', help='foo help') @@ -353,16 +197,7 @@ Most calls to the :class:`ArgumentParser` constructor will use the ``description=`` keyword argument. This argument gives a brief description of what the program does and how it works. In help messages, the description is displayed between the command-line usage string and the help messages for the -various arguments:: - - >>> parser = argparse.ArgumentParser(description='A foo that bars') - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - options: - -h, --help show this help message and exit +various arguments. By default, the description will be line-wrapped so that it fits within the given space. To change this behavior, see the formatter_class_ argument. @@ -492,7 +327,7 @@ should not be line-wrapped:: -h, --help show this help message and exit :class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, -including argument descriptions. However, multiple new lines are replaced with +including argument descriptions. However, multiple newlines are replaced with one. If you wish to preserve multiple blank lines, add spaces between the newlines. @@ -586,8 +421,8 @@ arguments will never be treated as file references. .. versionchanged:: 3.12 :class:`ArgumentParser` changed encoding and errors to read arguments files - from default (e.g. :func:`locale.getpreferredencoding(False) ` and - ``"strict"``) to :term:`filesystem encoding and error handler`. + from default (e.g. :func:`locale.getpreferredencoding(False) ` + and ``"strict"``) to the :term:`filesystem encoding and error handler`. Arguments file should be encoded in UTF-8 instead of ANSI Codepage on Windows. @@ -673,25 +508,8 @@ add_help ^^^^^^^^ By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed: - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help +the parser's help message. If ``-h`` or ``--help`` is supplied at the command +line, the ArgumentParser help will be printed. Occasionally, it may be useful to disable the addition of this help option. This can be achieved by passing ``False`` as the ``add_help=`` argument to @@ -723,7 +541,8 @@ exit_on_error ^^^^^^^^^^^^^ Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` -method of an :class:`ArgumentParser`, it will exit with error info. +method of an :class:`ArgumentParser`, it will print a *message* to :data:`sys.stderr` and exit with a status +code of 2. If the user would like to catch errors manually, the feature can be enabled by setting ``exit_on_error`` to ``False``:: @@ -783,7 +602,7 @@ The add_argument() method The following sections describe how each of these are used. -.. _name_or_flags: +.. _`name or flags`: name or flags ^^^^^^^^^^^^^ @@ -817,6 +636,25 @@ be positional:: usage: PROG [-h] [-f FOO] bar PROG: error: the following arguments are required: bar +By default, argparse automatically handles the internal naming and +display names of arguments, simplifying the process without requiring +additional configuration. +As such, you do not need to specify the dest_ and metavar_ parameters. +The dest_ parameter defaults to the argument name with underscores ``_`` +replacing hyphens ``-`` . The metavar_ parameter defaults to the +upper-cased name. For example:: + + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('--foo-bar') + >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + Namespace(foo_bar='FOO-BAR') + >>> parser.print_help() + usage: [-h] [--foo-bar FOO-BAR] + + optional arguments: + -h, --help show this help message and exit + --foo-bar FOO-BAR + .. _action: @@ -830,12 +668,7 @@ them, though most actions simply add an attribute to the object returned by how the command-line arguments should be handled. The supplied actions are: * ``'store'`` - This just stores the argument's value. This is the default - action. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args('--foo 1'.split()) - Namespace(foo='1') + action. * ``'store_const'`` - This stores the value specified by the const_ keyword argument; note that the const_ keyword argument defaults to ``None``. The @@ -850,7 +683,7 @@ how the command-line arguments should be handled. The supplied actions are: * ``'store_true'`` and ``'store_false'`` - These are special cases of ``'store_const'`` used for storing the values ``True`` and ``False`` respectively. In addition, they create default values of ``False`` and - ``True`` respectively. For example:: + ``True`` respectively:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -918,6 +751,9 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 +Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'`` or ``'extend'``) can be used with positional arguments. + You may also specify an arbitrary action by passing an Action subclass or other object that implements the same interface. The ``BooleanOptionalAction`` is available in ``argparse`` and adds support for boolean actions such as @@ -1045,6 +881,8 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: If the ``nargs`` keyword argument is not provided, the number of arguments consumed is determined by the action_. Generally this means a single command-line argument will be consumed and a single item (not a list) will be produced. +Actions that do not consume command-line arguments (e.g. +``'store_const'``) set ``nargs=0``. .. _const: @@ -1095,7 +933,7 @@ was not present at the command line:: Namespace(foo=42) If the target namespace already has an attribute set, the action *default* -will not over write it:: +will not overwrite it:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default=42) @@ -1169,7 +1007,6 @@ Common built-in types and functions can be used as type converters: parser.add_argument('distance', type=float) parser.add_argument('street', type=ascii) parser.add_argument('code_point', type=ord) - parser.add_argument('source_file', type=open) parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1')) parser.add_argument('datapath', type=pathlib.Path) @@ -1200,10 +1037,11 @@ better reporting than can be given by the ``type`` keyword. A :exc:`FileNotFoundError` exception would not be handled at all. Even :class:`~argparse.FileType` has its limitations for use with the ``type`` -keyword. If one argument uses *FileType* and then a subsequent argument fails, -an error is reported but the file is not automatically closed. In this case, it -would be better to wait until after the parser has run and then use the -:keyword:`with`-statement to manage the files. +keyword. If one argument uses :class:`~argparse.FileType` and then a +subsequent argument fails, an error is reported but the file is not +automatically closed. In this case, it would be better to wait until after +the parser has run and then use the :keyword:`with`-statement to manage the +files. For type checkers that simply check against a fixed set of values, consider using the choices_ keyword instead. @@ -1231,15 +1069,7 @@ if the argument was not one of the acceptable values:: Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified:: - - >>> parser = argparse.ArgumentParser(prog='doors.py') - >>> parser.add_argument('door', type=int, choices=range(1, 4)) - >>> print(parser.parse_args(['3'])) - Namespace(door=3) - >>> parser.parse_args(['4']) - usage: doors.py [-h] {1,2,3} - doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) +sequence should match the type_ specified. Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. @@ -1289,22 +1119,7 @@ help The ``help`` value is a string containing a brief description of the argument. When a user requests help (usually by using ``-h`` or ``--help`` at the command line), these ``help`` descriptions will be displayed with each -argument:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', action='store_true', - ... help='foo the bars before frobbling') - >>> parser.add_argument('bar', nargs='+', - ... help='one of the bars to be frobbled') - >>> parser.parse_args(['-h']) - usage: frobble [-h] [--foo] bar [bar ...] - - positional arguments: - bar one of the bars to be frobbled - - options: - -h, --help show this help message and exit - --foo foo the bars before frobbling +argument. The ``help`` strings can include various format specifiers to avoid repetition of things like the program name or the argument default_. The available @@ -1485,40 +1300,41 @@ this API may be passed as the ``action`` parameter to type=None, choices=None, required=False, help=None, \ metavar=None) -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` -except for the ``action`` itself. + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The Action class must accept the two positional arguments + plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` + except for the ``action`` itself. -Instances of Action (or return value of any callable to the ``action`` -parameter) should have attributes "dest", "option_strings", "default", "type", -"required", "help", etc. defined. The easiest way to ensure these attributes -are defined is to call ``Action.__init__``. + Instances of Action (or return value of any callable to the ``action`` + parameter) should have attributes "dest", "option_strings", "default", "type", + "required", "help", etc. defined. The easiest way to ensure these attributes + are defined is to call ``Action.__init__``. -Action instances should be callable, so subclasses must override the -``__call__`` method, which should accept four parameters: + Action instances should be callable, so subclasses must override the + ``__call__`` method, which should accept four parameters: -* ``parser`` - The ArgumentParser object which contains this action. + * *parser* - The ArgumentParser object which contains this action. -* ``namespace`` - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. + * *namespace* - The :class:`Namespace` object that will be returned by + :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this + object using :func:`setattr`. -* ``values`` - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. + * *values* - The associated command-line arguments, with any type conversions + applied. Type conversions are specified with the type_ keyword argument to + :meth:`~ArgumentParser.add_argument`. -* ``option_string`` - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. + * *option_string* - The option string that was used to invoke this action. + The ``option_string`` argument is optional, and will be absent if the action + is associated with a positional argument. -The ``__call__`` method may perform arbitrary actions, but will typically set -attributes on the ``namespace`` based on ``dest`` and ``values``. + The ``__call__`` method may perform arbitrary actions, but will typically set + attributes on the ``namespace`` based on ``dest`` and ``values``. + + Action subclasses can define a ``format_usage`` method that takes no argument + and return a string which will be used when printing the usage of the program. + If such method is not provided, a sensible default will be used. -Action subclasses can define a ``format_usage`` method that takes no argument -and return a string which will be used when printing the usage of the program. -If such method is not provided, a sensible default will be used. The parse_args() method ----------------------- @@ -1713,29 +1529,29 @@ The Namespace object Simple class used by default by :meth:`~ArgumentParser.parse_args` to create an object holding attributes and return it. -This class is deliberately simple, just an :class:`object` subclass with a -readable string representation. If you prefer to have dict-like view of the -attributes, you can use the standard Python idiom, :func:`vars`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> args = parser.parse_args(['--foo', 'BAR']) - >>> vars(args) - {'foo': 'BAR'} - -It may also be useful to have an :class:`ArgumentParser` assign attributes to an -already existing object, rather than a new :class:`Namespace` object. This can -be achieved by specifying the ``namespace=`` keyword argument:: + This class is deliberately simple, just an :class:`object` subclass with a + readable string representation. If you prefer to have dict-like view of the + attributes, you can use the standard Python idiom, :func:`vars`:: - >>> class C: - ... pass - ... - >>> c = C() - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) - >>> c.foo - 'BAR' + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> args = parser.parse_args(['--foo', 'BAR']) + >>> vars(args) + {'foo': 'BAR'} + + It may also be useful to have an :class:`ArgumentParser` assign attributes to an + already existing object, rather than a new :class:`Namespace` object. This can + be achieved by specifying the ``namespace=`` keyword argument:: + + >>> class C: + ... pass + ... + >>> c = C() + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) + >>> c.foo + 'BAR' Other utilities @@ -1749,12 +1565,12 @@ Sub-commands [option_strings], [dest], [required], \ [help], [metavar]) - Many programs split up their functionality into a number of sub-commands, - for example, the ``svn`` program can invoke sub-commands like ``svn + Many programs split up their functionality into a number of subcommands, + for example, the ``svn`` program can invoke subcommands like ``svn checkout``, ``svn update``, and ``svn commit``. Splitting up functionality this way can be a particularly good idea when a program performs several different functions which require different kinds of command-line arguments. - :class:`ArgumentParser` supports the creation of such sub-commands with the + :class:`ArgumentParser` supports the creation of such subcommands with the :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally called with no arguments and returns a special action object. This object has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a @@ -1763,18 +1579,18 @@ Sub-commands Description of parameters: - * title - title for the sub-parser group in help output; by default + * *title* - title for the sub-parser group in help output; by default "subcommands" if description is provided, otherwise uses title for positional arguments - * description - description for the sub-parser group in help output, by + * *description* - description for the sub-parser group in help output, by default ``None`` - * prog - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with sub-command help, by default the name of the program and any positional arguments before the subparser argument - * parser_class - class which will be used to create sub-parser instances, by + * *parser_class* - class which will be used to create sub-parser instances, by default the class of the current parser (e.g. ArgumentParser) * action_ - the basic type of action to be taken when this argument is @@ -1788,15 +1604,15 @@ Sub-commands * help_ - help for sub-parser group in help output, by default ``None`` - * metavar_ - string presenting available sub-commands in help; by default it - is ``None`` and presents sub-commands in form {cmd1, cmd2, ..} + * metavar_ - string presenting available subcommands in help; by default it + is ``None`` and presents subcommands in form {cmd1, cmd2, ..} Some example usage:: >>> # create the top-level parser >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo', action='store_true', help='foo help') - >>> subparsers = parser.add_subparsers(help='sub-command help') + >>> subparsers = parser.add_subparsers(help='subcommand help') >>> >>> # create the parser for the "a" command >>> parser_a = subparsers.add_parser('a', help='a help') @@ -1804,7 +1620,7 @@ Sub-commands >>> >>> # create the parser for the "b" command >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') + >>> parser_b.add_argument('--baz', choices=('X', 'Y', 'Z'), help='baz help') >>> >>> # parse some argument lists >>> parser.parse_args(['a', '12']) @@ -1831,7 +1647,7 @@ Sub-commands usage: PROG [-h] [--foo] {a,b} ... positional arguments: - {a,b} sub-command help + {a,b} subcommand help a a help b b help @@ -1902,12 +1718,12 @@ Sub-commands .. versionadded:: 3.13 - One particularly effective way of handling sub-commands is to combine the use + One particularly effective way of handling subcommands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For example:: - >>> # sub-command functions + >>> # subcommand functions >>> def foo(args): ... print(args.x * args.y) ... @@ -2189,20 +2005,20 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) -Sometimes a script may only parse a few of the command-line arguments, passing -the remaining arguments on to another script or program. In these cases, the -:meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like -:meth:`~ArgumentParser.parse_args` except that it does not produce an error when -extra arguments are present. Instead, it returns a two item tuple containing -the populated namespace and the list of remaining argument strings. + Sometimes a script may only parse a few of the command-line arguments, passing + the remaining arguments on to another script or program. In these cases, the + :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like + :meth:`~ArgumentParser.parse_args` except that it does not produce an error when + extra arguments are present. Instead, it returns a two item tuple containing + the populated namespace and the list of remaining argument strings. -:: + :: - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('bar') - >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) - (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action='store_true') + >>> parser.add_argument('bar') + >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) + (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) .. warning:: :ref:`Prefix matching ` rules apply to @@ -2260,90 +2076,38 @@ Intermixed parsing .. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None) .. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None) -A number of Unix commands allow the user to intermix optional arguments with -positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` -and :meth:`~ArgumentParser.parse_known_intermixed_args` methods -support this parsing style. - -These parsers do not support all the argparse features, and will raise -exceptions if unsupported features are used. In particular, subparsers, -and mutually exclusive groups that include both -optionals and positionals are not supported. - -The following example shows the difference between -:meth:`~ArgumentParser.parse_known_args` and -:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', -'3']`` as unparsed arguments, while the latter collects all the positionals -into ``rest``. :: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.add_argument('cmd') - >>> parser.add_argument('rest', nargs='*', type=int) - >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) - (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) - >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) - Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) - -:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple -containing the populated namespace and the list of remaining argument strings. -:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any -remaining unparsed argument strings. - -.. versionadded:: 3.7 - -.. _upgrading-optparse-code: - -Upgrading optparse code ------------------------ - -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support the new -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the standard library :mod:`optparse` -module in a number of ways including: + A number of Unix commands allow the user to intermix optional arguments with + positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` + and :meth:`~ArgumentParser.parse_known_intermixed_args` methods + support this parsing style. -* Handling positional arguments. -* Supporting sub-commands. -* Allowing alternative option prefixes like ``+`` and ``/``. -* Handling zero-or-more and one-or-more style arguments. -* Producing more informative usage messages. -* Providing a much simpler interface for custom ``type`` and ``action``. + These parsers do not support all the argparse features, and will raise + exceptions if unsupported features are used. In particular, subparsers, + and mutually exclusive groups that include both + optionals and positionals are not supported. -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: + The following example shows the difference between + :meth:`~ArgumentParser.parse_known_args` and + :meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', + '3']`` as unparsed arguments, while the latter collects all the positionals + into ``rest``. :: -* Replace all :meth:`optparse.OptionParser.add_option` calls with - :meth:`ArgumentParser.add_argument` calls. - -* Replace ``(options, args) = parser.parse_args()`` with ``args = - parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` - calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in the :mod:`argparse` context is called ``args``. - -* Replace :meth:`optparse.OptionParser.disable_interspersed_args` - by using :meth:`~ArgumentParser.parse_intermixed_args` instead of - :meth:`~ArgumentParser.parse_args`. - -* Replace callback actions and the ``callback_*`` keyword arguments with - ``type`` or ``action`` arguments. - -* Replace string names for ``type`` keyword arguments with the corresponding - type objects (e.g. int, float, complex, etc). + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.add_argument('cmd') + >>> parser.add_argument('rest', nargs='*', type=int) + >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) + (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) + >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) + Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) -* Replace :class:`optparse.Values` with :class:`Namespace` and - :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with - :exc:`ArgumentError`. + :meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple + containing the populated namespace and the list of remaining argument strings. + :meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any + remaining unparsed argument strings. -* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with - the standard Python syntax to use dictionaries to format strings, that is, - ``%(default)s`` and ``%(prog)s``. + .. versionadded:: 3.7 -* Replace the OptionParser constructor ``version`` argument with a call to - ``parser.add_argument('--version', action='version', version='')``. Exceptions ---------- @@ -2358,3 +2122,12 @@ Exceptions .. exception:: ArgumentTypeError Raised when something goes wrong converting a command line string to a type. + + +.. rubric:: Guides and Tutorials + +.. toctree:: + :maxdepth: 1 + + ../howto/argparse.rst + ../howto/argparse-optparse.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 55007624c876fa..3d2df035a85c21 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -178,9 +178,9 @@ Root nodes A Python module, as with :ref:`file input `. Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - *body* is a :class:`list` of the module's :ref:`ast-statements`. + ``body`` is a :class:`list` of the module's :ref:`ast-statements`. - *type_ignores* is a :class:`list` of the module's type ignore comments; + ``type_ignores`` is a :class:`list` of the module's type ignore comments; see :func:`ast.parse` for more details. .. doctest:: @@ -199,7 +199,7 @@ Root nodes A single Python :ref:`expression input `. Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - *body* is a single node, + ``body`` is a single node, one of the :ref:`expression types `. .. doctest:: @@ -214,7 +214,7 @@ Root nodes A single :ref:`interactive input `, like in :ref:`tut-interac`. Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - *body* is a :class:`list` of :ref:`statement nodes `. + ``body`` is a :class:`list` of :ref:`statement nodes `. .. doctest:: @@ -243,9 +243,9 @@ Root nodes # type: (int, int) -> int return a + b - *argtypes* is a :class:`list` of :ref:`expression nodes `. + ``argtypes`` is a :class:`list` of :ref:`expression nodes `. - *returns* is a single :ref:`expression node `. + ``returns`` is a single :ref:`expression node `. .. doctest:: @@ -902,7 +902,7 @@ Statements (indicating a "simple" target). A "simple" target consists solely of a :class:`Name` node that does not appear between parentheses; all other targets are considered complex. Only simple targets appear in - the :attr:`__annotations__` dictionary of modules and classes. + the :attr:`~object.__annotations__` dictionary of modules and classes. .. doctest:: @@ -1771,9 +1771,9 @@ aliases. .. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. *name* is the name of the type variable. - *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. *default_value* + A :class:`typing.TypeVar`. ``name`` is the name of the type variable. + ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. ``default_value`` is the default value; if the :class:`!TypeVar` has no default, this attribute will be set to ``None``. @@ -1801,8 +1801,8 @@ aliases. .. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. - *default_value* is the default value; if the :class:`!ParamSpec` has no default, + A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + ``default_value`` is the default value; if the :class:`!ParamSpec` has no default, this attribute will be set to ``None``. .. doctest:: @@ -1836,8 +1836,8 @@ aliases. .. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. - *default_value* is the default value; if the :class:`!TypeVarTuple` has no + A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + ``default_value`` is the default value; if the :class:`!TypeVarTuple` has no default, this attribute will be set to ``None``. .. doctest:: @@ -2491,7 +2491,7 @@ effects on the compilation of a program: differ in whitespace or similar details. Attributes include line numbers and column offsets. - .. versionadded:: 3.14 + .. versionadded:: next .. _ast-cli: diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 8312e55126a7c5..28d5aaf3692baa 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -24,11 +24,13 @@ Running an asyncio Program .. function:: run(coro, *, debug=None, loop_factory=None) - Execute the :term:`coroutine` *coro* and return the result. + Execute *coro* in an asyncio event loop and return the result. - This function runs the passed coroutine, taking care of - managing the asyncio event loop, *finalizing asynchronous - generators*, and closing the executor. + The argument can be any awaitable object. + + This function runs the awaitable, taking care of managing the + asyncio event loop, *finalizing asynchronous generators*, and + closing the executor. This function cannot be called when another asyncio event loop is running in the same thread. @@ -70,6 +72,10 @@ Running an asyncio Program Added *loop_factory* parameter. + .. versionchanged:: 3.14 + + *coro* can be any awaitable object. + Runner context manager ====================== @@ -104,17 +110,25 @@ Runner context manager .. method:: run(coro, *, context=None) - Run a :term:`coroutine ` *coro* in the embedded loop. + Execute *coro* in the embedded event loop. + + The argument can be any awaitable object. - Return the coroutine's result or raise its exception. + If the argument is a coroutine, it is wrapped in a Task. An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *coro* to run in. - The runner's default context is used if ``None``. + custom :class:`contextvars.Context` for the code to run in. + The runner's default context is used if context is ``None``. + + Returns the awaitable's result or raises an exception. This function cannot be called when another asyncio event loop is running in the same thread. + .. versionchanged:: 3.14 + + *coro* can be any awaitable object. + .. method:: close() Close the runner. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 4716a3f9c8ac79..f27e858cf420f4 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -158,7 +158,7 @@ other coroutines:: # Nothing happens if we just call "nested()". # A coroutine object is created but not awaited, # so it *won't run at all*. - nested() + nested() # will raise a "RuntimeWarning". # Let's do it differently now and await it: print(await nested()) # will print "42". diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index d5876054da3eee..eafc038d6cb722 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -393,13 +393,22 @@ The :mod:`calendar` module exports the following data attributes: .. data:: day_name - An array that represents the days of the week in the current locale. + A sequence that represents the days of the week in the current locale, + where Monday is day number 0. + + >>> import calendar + >>> list(calendar.day_name) + ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] .. data:: day_abbr - An array that represents the abbreviated days of the week in the current locale. + A sequence that represents the abbreviated days of the week in the current locale, + where Mon is day number 0. + >>> import calendar + >>> list(calendar.day_abbr) + ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] .. data:: MONDAY TUESDAY @@ -426,17 +435,24 @@ The :mod:`calendar` module exports the following data attributes: .. data:: month_name - An array that represents the months of the year in the current locale. This + A sequence that represents the months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_name[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_name) + ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. data:: month_abbr - An array that represents the abbreviated months of the year in the current + A sequence that represents the abbreviated months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_abbr) + ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] .. data:: JANUARY FEBRUARY diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cee4e350c498fe..0cc9063f153aba 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -874,8 +874,8 @@ they add the ability to access fields by name instead of position index. ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to ``1``, and ``z`` will default to ``2``. - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. + If *module* is defined, the :attr:`~type.__module__` attribute of the + named tuple is set to that value. Named tuple instances do not have per-instance dictionaries, so they are lightweight and require no more memory than regular tuples. diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index e3b24451188cc4..ce72127127c7a6 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -286,14 +286,6 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. - .. note:: - The default :mod:`multiprocessing` start method - (see :ref:`multiprocessing-start-methods`) will change away from - *fork* in Python 3.14. Code that requires *fork* be used for their - :class:`ProcessPoolExecutor` should explicitly specify that by - passing a ``mp_context=multiprocessing.get_context("fork")`` - parameter. - .. versionchanged:: 3.11 The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. @@ -310,6 +302,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *max_workers* uses :func:`os.process_cpu_count` by default, instead of :func:`os.cpu_count`. + .. versionchanged:: 3.14 + The default process start method (see + :ref:`multiprocessing-start-methods`) changed away from *fork*. If you + require the *fork* start method for :class:`ProcessPoolExecutor` you must + explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. _processpoolexecutor-example: ProcessPoolExecutor Example diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index b5c18bbccffb78..3aad6f7b5d2d20 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -54,6 +54,7 @@ can be customized by end users easily. import os os.remove("example.ini") + os.remove("override.ini") Quick Start diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 2a79dfe8f81e26..2b1fb9fdd29cd8 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -144,51 +144,89 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. - Every thread will have a different top-level :class:`~contextvars.Context` - object. This means that a :class:`ContextVar` object behaves in a similar - fashion to :func:`threading.local` when values are assigned in different - threads. + Each thread has its own effective stack of :class:`!Context` objects. The + :term:`current context` is the :class:`!Context` object at the top of the + current thread's stack. All :class:`!Context` objects in the stacks are + considered to be *entered*. + + *Entering* a context, which can be done by calling its :meth:`~Context.run` + method, makes the context the current context by pushing it onto the top of + the current thread's context stack. + + *Exiting* from the current context, which can be done by returning from the + callback passed to the :meth:`~Context.run` method, restores the current + context to what it was before the context was entered by popping the context + off the top of the context stack. + + Since each thread has its own context stack, :class:`ContextVar` objects + behave in a similar fashion to :func:`threading.local` when values are + assigned in different threads. + + Attempting to enter an already entered context, including contexts entered in + other threads, raises a :exc:`RuntimeError`. + + After exiting a context, it can later be re-entered (from any thread). + + Any changes to :class:`ContextVar` values via the :meth:`ContextVar.set` + method are recorded in the current context. The :meth:`ContextVar.get` + method returns the value associated with the current context. Exiting a + context effectively reverts any changes made to context variables while the + context was entered (if needed, the values can be restored by re-entering the + context). Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) - Execute ``callable(*args, **kwargs)`` code in the context object - the *run* method is called on. Return the result of the execution - or propagate an exception if one occurred. + Enters the Context, executes ``callable(*args, **kwargs)``, then exits the + Context. Returns *callable*'s return value, or propagates an exception if + one occurred. + + Example: + + .. testcode:: + + import contextvars - Any changes to any context variables that *callable* makes will - be contained in the context object:: + var = contextvars.ContextVar('var') + var.set('spam') + print(var.get()) # 'spam' - var = ContextVar('var') - var.set('spam') + ctx = contextvars.copy_context() - def main(): - # 'var' was set to 'spam' before - # calling 'copy_context()' and 'ctx.run(main)', so: - # var.get() == ctx[var] == 'spam' + def main(): + # 'var' was set to 'spam' before + # calling 'copy_context()' and 'ctx.run(main)', so: + print(var.get()) # 'spam' + print(ctx[var]) # 'spam' - var.set('ham') + var.set('ham') - # Now, after setting 'var' to 'ham': - # var.get() == ctx[var] == 'ham' + # Now, after setting 'var' to 'ham': + print(var.get()) # 'ham' + print(ctx[var]) # 'ham' - ctx = copy_context() + # Any changes that the 'main' function makes to 'var' + # will be contained in 'ctx'. + ctx.run(main) - # Any changes that the 'main' function makes to 'var' - # will be contained in 'ctx'. - ctx.run(main) + # The 'main()' function was run in the 'ctx' context, + # so changes to 'var' are contained in it: + print(ctx[var]) # 'ham' - # The 'main()' function was run in the 'ctx' context, - # so changes to 'var' are contained in it: - # ctx[var] == 'ham' + # However, outside of 'ctx', 'var' is still set to 'spam': + print(var.get()) # 'spam' - # However, outside of 'ctx', 'var' is still set to 'spam': - # var.get() == 'spam' + .. testoutput:: + :hide: - The method raises a :exc:`RuntimeError` when called on the same - context object from more than one OS thread, or when called - recursively. + spam + spam + spam + ham + ham + ham + spam .. method:: copy() diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index a218304653aee9..d76b8d4809c078 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2303,7 +2303,7 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`double complex` datatype, if available. The constructor accepts an optional :class:`complex` initializer. - .. versionadded:: 3.14 + .. versionadded:: next .. class:: c_float_complex @@ -2589,6 +2589,8 @@ fields, or any other data types containing pointer type fields. the structure when being packed or unpacked to/from memory. Setting this attribute to 0 is the same as not setting it at all. + .. versionadded:: 3.13 + .. attribute:: _layout_ An optional string naming the struct/union layout. It can currently diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index cfca11afbd2e41..e34b2db0210960 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -187,13 +187,6 @@ Module contents If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` is raised. - .. warning:: - Calling no-arg :func:`super` in dataclasses using ``slots=True`` - will result in the following exception being raised: - ``TypeError: super(type, obj): obj must be an instance or subtype of type``. - The two-arg :func:`super` is a valid workaround. - See :gh:`90562` for full details. - .. warning:: Passing parameters to a base class :meth:`~object.__init_subclass__` when using ``slots=True`` will result in a :exc:`TypeError`. @@ -238,7 +231,7 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that @@ -307,6 +300,10 @@ Module contents .. versionadded:: 3.10 + - ``doc``: optional docstring for this field. + + .. versionadded:: 3.13 + If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not @@ -402,7 +399,7 @@ Module contents :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) Creates a new dataclass with name *cls_name*, fields as defined in *fields*, base classes as given in *bases*, and initialized @@ -418,6 +415,11 @@ Module contents of the dataclass is set to that value. By default, it is set to the module name of the caller. + The *decorator* parameter is a callable that will be used to create the dataclass. + It should take the class object as a first argument and the same keyword arguments + as :func:`@dataclass `. By default, the :func:`@dataclass ` + function is used. + This function is not strictly required, because any Python mechanism for creating a new class with :attr:`!__annotations__` can then apply the :func:`@dataclass ` function to convert that class to @@ -441,6 +443,9 @@ Module contents def add_one(self): return self.x + 1 + .. versionadded:: 3.14 + Added the *decorator* parameter. + .. function:: replace(obj, /, **changes) Creates a new object of the same type as *obj*, replacing diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 0e7dc4f262bab4..2f81080d525f86 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -180,19 +180,19 @@ Objects of the :class:`date` type are always naive. An object of type :class:`.time` or :class:`.datetime` may be aware or naive. -A :class:`.datetime` object *d* is aware if both of the following hold: +A :class:`.datetime` object ``d`` is aware if both of the following hold: 1. ``d.tzinfo`` is not ``None`` 2. ``d.tzinfo.utcoffset(d)`` does not return ``None`` -Otherwise, *d* is naive. +Otherwise, ``d`` is naive. -A :class:`.time` object *t* is aware if both of the following hold: +A :class:`.time` object ``t`` is aware if both of the following hold: 1. ``t.tzinfo`` is not ``None`` 2. ``t.tzinfo.utcoffset(None)`` does not return ``None``. -Otherwise, *t* is naive. +Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. @@ -295,6 +295,20 @@ Instance attributes (read-only): Between 0 and 86,399 inclusive. + .. caution:: + + It is a somewhat common bug for code to unintentionally use this attribute + when it is actually intended to get a :meth:`~timedelta.total_seconds` + value instead: + + .. doctest:: + + >>> from datetime import timedelta + >>> duration = timedelta(seconds=11235813) + >>> duration.days, duration.seconds + (130, 3813) + >>> duration.total_seconds() + 11235813.0 .. attribute:: timedelta.microseconds @@ -344,14 +358,14 @@ Supported operations: +--------------------------------+-----------------------------------------------+ | ``q, r = divmod(t1, t2)`` | Computes the quotient and the remainder: | | | ``q = t1 // t2`` (3) and ``r = t1 % t2``. | -| | q is an integer and r is a :class:`timedelta` | -| | object. | +| | ``q`` is an integer and ``r`` is a | +| | :class:`timedelta` object. | +--------------------------------+-----------------------------------------------+ | ``+t1`` | Returns a :class:`timedelta` object with the | | | same value. (2) | +--------------------------------+-----------------------------------------------+ | ``-t1`` | Equivalent to ``timedelta(-t1.days, | -| | -t1.seconds*, -t1.microseconds)``, | +| | -t1.seconds, -t1.microseconds)``, | | | and to ``t1 * -1``. (1)(4) | +--------------------------------+-----------------------------------------------+ | ``abs(t)`` | Equivalent to ``+t`` when ``t.days >= 0``, | @@ -512,7 +526,7 @@ Other constructors, all class methods: January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, + date.max.toordinal()``. For any date ``d``, ``date.fromordinal(d.toordinal()) == d``. @@ -548,6 +562,39 @@ Other constructors, all class methods: .. versionadded:: 3.8 +.. classmethod:: date.strptime(date_string, format) + + Return a :class:`.date` corresponding to *date_string*, parsed according to + *format*. This is equivalent to:: + + date(*(time.strptime(date_string, format)[0:3])) + + :exc:`ValueError` is raised if the date_string and format + can't be parsed by :func:`time.strptime` or if it returns a value which isn't a + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`date.fromisoformat`. + + .. note:: + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import date + >>> date_string = "02/29" + >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' + + .. versionadded:: 3.14 + Class attributes: @@ -683,7 +730,7 @@ Instance methods: .. method:: date.toordinal() Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, + has ordinal 1. For any :class:`date` object ``d``, ``date.fromordinal(d.toordinal()) == d``. @@ -735,7 +782,7 @@ Instance methods: .. method:: date.__str__() - For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``. + For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. .. method:: date.ctime() @@ -901,6 +948,10 @@ Other constructors, all class methods: This function is preferred over :meth:`today` and :meth:`utcnow`. + .. note:: + + Subsequent calls to :meth:`!datetime.now` may return the same + instant depending on the precision of the underlying clock. .. classmethod:: datetime.utcnow() @@ -1012,7 +1063,7 @@ Other constructors, all class methods: is used. If the *date* argument is a :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. - For any :class:`.datetime` object *d*, + For any :class:`.datetime` object ``d``, ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. .. versionchanged:: 3.6 @@ -1219,11 +1270,11 @@ Supported operations: If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments + object ``t`` such that ``datetime2 + t == datetime1``. No time zone adjustments are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes. The + as if ``a`` and ``b`` were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. @@ -1403,11 +1454,11 @@ Instance methods: .. method:: datetime.utctimetuple() - If :class:`.datetime` instance *d* is naive, this is the same as + If :class:`.datetime` instance ``d`` is naive, this is the same as ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. - If *d* is aware, *d* is normalized to UTC time, by subtracting + If ``d`` is aware, ``d`` is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note that an :exc:`OverflowError` may be raised if ``d.year`` was @@ -1555,7 +1606,7 @@ Instance methods: .. method:: datetime.__str__() - For a :class:`.datetime` instance *d*, ``str(d)`` is equivalent to + For a :class:`.datetime` instance ``d``, ``str(d)`` is equivalent to ``d.isoformat(' ')``. @@ -1802,7 +1853,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 :class:`.time` objects support equality and order comparisons, -where *a* is considered less than *b* when *a* precedes *b* in time. +where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. Naive and aware :class:`!time` objects are never equal. Order comparison between naive and aware :class:`!time` objects raises @@ -1827,7 +1878,7 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. details. -Other constructor: +Other constructors: .. classmethod:: time.fromisoformat(time_string) @@ -1869,6 +1920,22 @@ Other constructor: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. +.. classmethod:: time.strptime(date_string, format) + + Return a :class:`.time` corresponding to *date_string*, parsed according to + *format*. + + If *format* does not contain microseconds or timezone information, this is equivalent to:: + + time(*(time.strptime(date_string, format)[3:6])) + + :exc:`ValueError` is raised if the *date_string* and *format* + cannot be parsed by :func:`time.strptime` or if it returns a value which is not a + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`time.fromisoformat`. + + .. versionadded:: 3.14 + Instance methods: @@ -1933,7 +2000,7 @@ Instance methods: .. method:: time.__str__() - For a time *t*, ``str(t)`` is equivalent to ``t.isoformat()``. + For a time ``t``, ``str(t)`` is equivalent to ``t.isoformat()``. .. method:: time.strftime(format) @@ -2367,24 +2434,22 @@ Class attributes: ``strftime(format)`` method, to create a string representing the time under the control of an explicit format string. -Conversely, the :meth:`datetime.strptime` class method creates a -:class:`.datetime` object from a string representing a date and time and a -corresponding format string. +Conversely, the :meth:`date.strptime`, :meth:`datetime.strptime` and +:meth:`time.strptime` class methods create an object from a string +representing the time and a corresponding format string. The table below provides a high-level comparison of :meth:`~.datetime.strftime` versus :meth:`~.datetime.strptime`: -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| | ``strftime`` | ``strptime`` | -+================+========================================================+==============================================================================+ -| Usage | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Type of method | Instance method | Class method | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Method of | :class:`date`; :class:`.datetime`; :class:`.time` | :class:`.datetime` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| | ``strftime`` | ``strptime`` | ++================+========================================================+============================================================+ +| Usage | Convert object to a string according to a given format | Parse a string into an object given a corresponding format | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| Type of method | Instance method | Class method | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ .. _format-codes: diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 916f17cadfaa7e..c9a3e448cad063 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -598,6 +598,23 @@ Decimal objects .. versionadded:: 3.1 + .. classmethod:: from_number(number) + + Alternative constructor that only accepts instances of + :class:`float`, :class:`int` or :class:`Decimal`, but not strings + or tuples. + + .. doctest:: + + >>> Decimal.from_number(314) + Decimal('314') + >>> Decimal.from_number(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) + Decimal('3.14') + + .. versionadded:: 3.14 + .. method:: fma(other, third, context=None) Fused multiply-add. Return self*other+third with no rounding of the diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index cad73192f7cd43..1d084a8bf38d98 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -959,7 +959,7 @@ iterations of the loop. list of constants supported by this instruction. Used by the :keyword:`assert` statement to load :exc:`AssertionError`. - .. versionadded:: 3.14 + .. versionadded:: next .. opcode:: LOAD_BUILD_CLASS @@ -969,7 +969,8 @@ iterations of the loop. .. opcode:: GET_LEN - Perform ``STACK.append(len(STACK[-1]))``. + Perform ``STACK.append(len(STACK[-1]))``. Used in :keyword:`match` statements where + comparison with structure of pattern is needed. .. versionadded:: 3.10 @@ -1434,7 +1435,7 @@ iterations of the loop. slot ``i`` of the "fast locals" storage in this mapping. If the name is not found there, loads it from the cell contained in slot ``i``, similar to :opcode:`LOAD_DEREF`. This is used for loading - free variables in class bodies (which previously used + :term:`closure variables ` in class bodies (which previously used :opcode:`!LOAD_CLASSDEREF`) and in :ref:`annotation scopes ` within class bodies. @@ -1463,8 +1464,8 @@ iterations of the loop. .. opcode:: COPY_FREE_VARS (n) - Copies the ``n`` free variables from the closure into the frame. - Removes the need for special code on the caller's side when calling + Copies the ``n`` :term:`free (closure) variables ` from the closure + into the frame. Removes the need for special code on the caller's side when calling closures. .. versionadded:: 3.11 @@ -1826,7 +1827,7 @@ iterations of the loop. If ``type(STACK[-1]).__xxx__`` is not a method, leave ``STACK[-1].__xxx__; NULL`` on the stack. - .. versionadded:: 3.14 + .. versionadded:: next **Pseudo-instructions** @@ -1872,6 +1873,12 @@ but are replaced by real opcodes or removed before bytecode is generated. Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. +.. opcode:: JUMP_IF_TRUE +.. opcode:: JUMP_IF_FALSE + + Conditional jumps which do not impact the stack. Replaced by the sequence + ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot ``i`` of the "fast locals" @@ -1931,10 +1938,10 @@ instructions: .. data:: hasfree - Sequence of bytecodes that access a free variable. 'free' in this - context refers to names in the current scope that are referenced by inner - scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes. + Sequence of bytecodes that access a :term:`free (closure) variable `. + 'free' in this context refers to names in the current scope that are + referenced by inner scopes or names in outer scopes that are referenced + from this scope. It does *not* include references to global or builtin scopes. .. data:: hasname diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 34121f8c0a7727..a86e227429b06d 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -58,11 +58,12 @@ * the type itself (``typ``) * the type's fully qualified name (``typ.__module__ + '.' + typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). + * the type's :attr:`qualname ` (``typ.__qualname__``) + * the type's :attr:`name ` (``typ.__name__``). If none of the above match, repeat all of the checks above for each of - the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key + the types in the :term:`MRO` (:attr:`typ.__mro__ `). + Finally, if no other key yields a handler, check for a handler for the key ``None``. If there is no handler for ``None``, raise a :exc:`KeyError` for the fully qualified name of the type. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index bcbd00c833e28e..7f8044932fae99 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -317,7 +317,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. class. When *use_default_map* is ``True`` (the default), the standard mapping of header names to classes is copied in to the registry during initialization. *base_class* is always the last class in the generated - class's ``__bases__`` list. + class's :class:`~type.__bases__` list. The default mappings are: diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 2ee154952549ac..fc7f9a6301a915 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -166,6 +166,16 @@ another rational number, or from a string. instance. + .. classmethod:: from_number(number) + + Alternative constructor which only accepts instances of + :class:`numbers.Integral`, :class:`numbers.Rational`, + :class:`float` or :class:`decimal.Decimal`, and objects with + the :meth:`!as_integer_ratio` method, but not strings. + + .. versionadded:: 3.14 + + .. method:: limit_denominator(max_denominator=1000000) Finds and returns the closest :class:`Fraction` to ``self`` that has diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index b2b0086437f1db..0638df04c6ff40 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -283,9 +283,11 @@ are always available. They are listed here in alphabetical order. :func:`property`. .. versionchanged:: 3.10 - Class methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and - have a new ``__wrapped__`` attribute. + Class methods now inherit the method attributes + (:attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__doc__` and + :attr:`~function.__annotations__`) and have a new ``__wrapped__`` + attribute. .. deprecated-removed:: 3.11 3.13 Class methods can no longer wrap other :term:`descriptors ` such as @@ -682,9 +684,10 @@ are always available. They are listed here in alphabetical order. ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. The *closure* argument specifies a closure--a tuple of cellvars. - It's only valid when the *object* is a code object containing free variables. - The length of the tuple must exactly match the number of free variables - referenced by the code object. + It's only valid when the *object* is a code object containing + :term:`free (closure) variables `. + The length of the tuple must exactly match the length of the code object's + :attr:`~codeobject.co_freevars` attribute. .. audit-event:: exec code_object exec @@ -1286,8 +1289,9 @@ are always available. They are listed here in alphabetical order. .. note:: - :class:`object` does *not* have a :attr:`~object.__dict__`, so you can't - assign arbitrary attributes to an instance of the :class:`object` class. + :class:`object` instances do *not* have :attr:`~object.__dict__` + attributes, so you can't assign arbitrary attributes to an instance of + :class:`object`. .. function:: oct(x) @@ -1907,10 +1911,11 @@ are always available. They are listed here in alphabetical order. For more information on static methods, see :ref:`types`. .. versionchanged:: 3.10 - Static methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``), - have a new ``__wrapped__`` attribute, and are now callable as regular - functions. + Static methods now inherit the method attributes + (:attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__doc__` and + :attr:`~function.__annotations__`), have a new ``__wrapped__`` attribute, + and are now callable as regular functions. .. index:: @@ -1961,11 +1966,11 @@ are always available. They are listed here in alphabetical order. to be searched. The search starts from the class right after the *type*. - For example, if :attr:`~class.__mro__` of *object_or_type* is + For example, if :attr:`~type.__mro__` of *object_or_type* is ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, then :func:`super` searches ``C -> A -> object``. - The :attr:`~class.__mro__` attribute of the class corresponding to + The :attr:`~type.__mro__` attribute of the class corresponding to *object_or_type* lists the method resolution search order used by both :func:`getattr` and :func:`super`. The attribute is dynamic and can change whenever the inheritance hierarchy is updated. @@ -2044,28 +2049,30 @@ are always available. They are listed here in alphabetical order. With one argument, return the type of an *object*. The return value is a type object and generally the same object as returned by - :attr:`object.__class__ `. + :attr:`object.__class__`. The :func:`isinstance` built-in function is recommended for testing the type of an object, because it takes subclasses into account. - With three arguments, return a new type object. This is essentially a dynamic form of the :keyword:`class` statement. The *name* string is - the class name and becomes the :attr:`~definition.__name__` attribute. + the class name and becomes the :attr:`~type.__name__` attribute. The *bases* tuple contains the base classes and becomes the - :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + :attr:`~type.__bases__` attribute; if empty, :class:`object`, the ultimate base of all classes, is added. The *dict* dictionary contains attribute and method definitions for the class body; it may be copied - or wrapped before becoming the :attr:`~object.__dict__` attribute. - The following two statements create identical :class:`type` objects: + or wrapped before becoming the :attr:`~type.__dict__` attribute. + The following two statements create identical :class:`!type` objects: >>> class X: ... a = 1 ... >>> X = type('X', (), dict(a=1)) - See also :ref:`bltin-type-objects`. + See also: + + * :ref:`Documentation on attributes and methods on classes `. + * :ref:`bltin-type-objects` Keyword arguments provided to the three argument form are passed to the appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) @@ -2075,18 +2082,18 @@ are always available. They are listed here in alphabetical order. See also :ref:`class-customization`. .. versionchanged:: 3.6 - Subclasses of :class:`type` which don't override ``type.__new__`` may no + Subclasses of :class:`!type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. .. function:: vars() vars(object) Return the :attr:`~object.__dict__` attribute for a module, class, instance, - or any other object with a :attr:`~object.__dict__` attribute. + or any other object with a :attr:`!__dict__` attribute. Objects such as modules and instances have an updateable :attr:`~object.__dict__` attribute; however, other objects may have write restrictions on their - :attr:`~object.__dict__` attributes (for example, classes use a + :attr:`!__dict__` attributes (for example, classes use a :class:`types.MappingProxyType` to prevent direct dictionary updates). Without an argument, :func:`vars` acts like :func:`locals`. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 008cde399baed2..e26a2226aa947a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -328,6 +328,14 @@ The :mod:`functools` module defines the following functions: Returning ``NotImplemented`` from the underlying comparison function for unrecognised types is now supported. +.. data:: Placeholder + + A singleton object used as a sentinel to reserve a place + for positional arguments when calling :func:`partial` + and :func:`partialmethod`. + + .. versionadded:: 3.14 + .. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called @@ -338,26 +346,68 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: def partial(func, /, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = {**keywords, **fkeywords} - return func(*args, *fargs, **newkeywords) + def newfunc(*more_args, **more_keywords): + return func(*args, *more_args, **(keywords | more_keywords)) newfunc.func = func newfunc.args = args newfunc.keywords = keywords return newfunc - The :func:`partial` is used for partial function application which "freezes" + The :func:`!partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument - defaults to two: + defaults to ``2``: + + .. doctest:: - >>> from functools import partial >>> basetwo = partial(int, base=2) >>> basetwo.__doc__ = 'Convert base 2 string to an int.' >>> basetwo('10010') 18 + If :data:`Placeholder` sentinels are present in *args*, they will be filled first + when :func:`!partial` is called. This makes it possible to pre-fill any positional + argument with a call to :func:`!partial`; without :data:`!Placeholder`, only the + first positional argument can be pre-filled. + + If any :data:`!Placeholder` sentinels are present, all must be filled at call time: + + .. doctest:: + + >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") + >>> say_to_world('Hello', 'dear') + Hello dear world! + + Calling ``say_to_world('Hello')`` raises a :exc:`TypeError`, because + only one positional argument is provided, but there are two placeholders + that must be filled in. + + If :func:`!partial` is applied to an existing :func:`!partial` object, + :data:`!Placeholder` sentinels of the input object are filled in with + new positional arguments. + A placeholder can be retained by inserting a new + :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: + + .. doctest:: + + >>> from functools import partial, Placeholder as _ + >>> remove = partial(str.replace, _, _, '') + >>> message = 'Hello, dear dear world!' + >>> remove(message, ' dear') + 'Hello, world!' + >>> remove_dear = partial(remove, _, ' dear') + >>> remove_dear(message) + 'Hello, world!' + >>> remove_first_dear = partial(remove_dear, _, 1) + >>> remove_first_dear(message) + 'Hello, dear world!' + + :data:`!Placeholder` has no special treatment when used in a keyword + argument to :func:`!partial`. + + .. versionchanged:: 3.14 + Added support for :data:`Placeholder` in positional arguments. .. class:: partialmethod(func, /, *args, **keywords) @@ -492,6 +542,25 @@ The :mod:`functools` module defines the following functions: ... print(arg.real, arg.imag) ... + For code that dispatches on a collections type (e.g., ``list``), but wants + to typehint the items of the collection (e.g., ``list[int]``), the + dispatch type should be passed explicitly to the decorator itself with the + typehint going into the function definition:: + + >>> @fun.register(list) + ... def _(arg: list[int], verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + .. note:: + + At runtime the function will dispatch on an instance of a list regardless + of the type contained within the list i.e. ``[1,2,3]`` will be + dispatched the same as ``["foo", "bar", "baz"]``. The annotation + provided in this example is for static type checkers only and has no + runtime impact. To enable registering :term:`lambdas` and pre-existing functions, the :func:`register` attribute can also be used in a functional form:: @@ -646,10 +715,11 @@ The :mod:`functools` module defines the following functions: attributes of the wrapper function are updated with the corresponding attributes from the original function. The default values for these arguments are the module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__``, - ``__type_params__``, and ``__doc__``, the documentation string) - and ``WRAPPER_UPDATES`` (which - updates the wrapper function's ``__dict__``, i.e. the instance dictionary). + function's :attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__annotations__`, + :attr:`~function.__type_params__`, and :attr:`~function.__doc__`, the + documentation string) and ``WRAPPER_UPDATES`` (which updates the wrapper + function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function @@ -670,7 +740,7 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.2 The ``__wrapped__`` attribute is now automatically added. - The ``__annotations__`` attribute is now copied by default. + The :attr:`~function.__annotations__` attribute is now copied by default. Missing attributes no longer trigger an :exc:`AttributeError`. .. versionchanged:: 3.4 @@ -679,7 +749,7 @@ The :mod:`functools` module defines the following functions: (see :issue:`17482`) .. versionchanged:: 3.12 - The ``__type_params__`` attribute is now copied by default. + The :attr:`~function.__type_params__` attribute is now copied by default. .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -741,9 +811,7 @@ have three read-only attributes: The keyword arguments that will be supplied when the :class:`partial` object is called. -:class:`partial` objects are like :class:`function` objects in that they are +:class:`partial` objects are like :ref:`function objects ` in that they are callable, weak referenceable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes -are not created automatically. Also, :class:`partial` objects defined in -classes behave like static methods and do not transform into bound methods -during instance attribute look-up. +differences. For instance, the :attr:`~definition.__name__` and :attr:`~definition.__doc__` attributes +are not created automatically. diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 6b6e158f6eba2c..f24e73517e5767 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -184,11 +184,12 @@ The module defines the following items: attribute instead. -.. function:: compress(data, compresslevel=9, *, mtime=None) +.. function:: compress(data, compresslevel=9, *, mtime=0) Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. + the :class:`GzipFile` constructor above, + but *mtime* defaults to 0 for reproducible output. .. versionadded:: 3.2 .. versionchanged:: 3.8 @@ -203,6 +204,10 @@ The module defines the following items: .. versionchanged:: 3.13 The gzip header OS byte is guaranteed to be set to 255 when this function is used as was the case in 3.10 and earlier. + .. versionchanged:: 3.14 + The *mtime* parameter now defaults to 0 for reproducible output. + For the previous behaviour of using the current time, + pass ``None`` to *mtime*. .. function:: decompress(data) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 66ba621084c898..85d5a2d684d6eb 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -100,6 +100,13 @@ You can also get a :ref:`distribution's version number `, list its :ref:`requirements`. +.. exception:: PackageNotFoundError + + Subclass of :class:`ModuleNotFoundError` raised by several functions in this + module when queried for a distribution package which is not installed in the + current Python environment. + + Functional API ============== @@ -111,31 +118,53 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a collection of entry points. -Entry points are represented by ``EntryPoint`` instances; -each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, -``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute. +.. function:: entry_points(**select_params) + + Returns a :class:`EntryPoints` instance describing entry points for the + current environment. Any given keyword parameters are passed to the + :meth:`!select` method for comparison to the attributes of + the individual entry point definitions. + + Note: it is not currently possible to query for entry points based on + their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution` + instances do not currently compare equal, even if they have the same attributes) + +.. class:: EntryPoints + + Details of a collection of installed entry points. + + Also provides a ``.groups`` attribute that reports all identifed entry + point groups, and a ``.names`` attribute that reports all identified entry + point names. + +.. class:: EntryPoint + + Details of an installed entry point. + + Each :class:`!EntryPoint` instance has ``.name``, ``.group``, and ``.value`` + attributes and a ``.load()`` method to resolve the value. There are also + ``.module``, ``.attr``, and ``.extras`` attributes for getting the + components of the ``.value`` attribute, and ``.dist`` for obtaining + information regarding the distribution package that provides the entry point. Query all entry points:: >>> eps = entry_points() # doctest: +SKIP -The ``entry_points()`` function returns an ``EntryPoints`` object, -a collection of all ``EntryPoint`` objects with ``names`` and ``groups`` +The :func:`!entry_points` function returns a :class:`!EntryPoints` object, +a collection of all :class:`!EntryPoint` objects with ``names`` and ``groups`` attributes for convenience:: >>> sorted(eps.groups) # doctest: +SKIP ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] -``EntryPoints`` has a ``select`` method to select entry points +:class:`!EntryPoints` has a :meth:`!select` method to select entry points matching specific properties. Select entry points in the ``console_scripts`` group:: >>> scripts = eps.select(group='console_scripts') # doctest: +SKIP -Equivalently, since ``entry_points`` passes keyword arguments +Equivalently, since :func:`!entry_points` passes keyword arguments through to select:: >>> scripts = entry_points(group='console_scripts') # doctest: +SKIP @@ -189,31 +218,41 @@ for more information on entry points, their definition, and usage. Distribution metadata --------------------- -Every `Distribution Package `_ includes some metadata, -which you can extract using the -``metadata()`` function:: +.. function:: metadata(distribution_name) + + Return the distribution metadata corresponding to the named + distribution package as a :class:`PackageMetadata` instance. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +.. class:: PackageMetadata + + A concrete implementation of the + `PackageMetadata protocol `_. + + In addition to providing the defined protocol methods and attributes, subscripting + the instance is equivalent to calling the :meth:`!get` method. + +Every `Distribution Package `_ +includes some metadata, which you can extract using the :func:`!metadata` function:: >>> wheel_metadata = metadata('wheel') # doctest: +SKIP -The keys of the returned data structure, a ``PackageMetadata``, -name the metadata keywords, and +The keys of the returned data structure name the metadata keywords, and the values are returned unparsed from the distribution metadata:: >>> wheel_metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' -``PackageMetadata`` also presents a ``json`` attribute that returns +:class:`PackageMetadata` also presents a :attr:`!json` attribute that returns all the metadata in a JSON-compatible form per :PEP:`566`:: >>> wheel_metadata.json['requires_python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' -.. note:: - - The actual type of the object returned by ``metadata()`` is an - implementation detail and should be accessed only through the interface - described by the - `PackageMetadata protocol `_. +The full set of available metadata is not described here. +See the PyPA `Core metadata specification `_ for additional details. .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented @@ -227,7 +266,15 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: Distribution versions --------------------- -The ``version()`` function is the quickest way to get a +.. function:: version(distribution_name) + + Return the installed distribution package version for the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +The :func:`!version` function is the quickest way to get a `Distribution Package `_'s version number, as a string:: @@ -240,12 +287,28 @@ number, as a string:: Distribution files ------------------ -You can also get the full set of files contained within a distribution. The -``files()`` function takes a `Distribution Package `_ name -and returns all of the -files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: +.. function:: files(distribution_name) + + Return the full set of files contained within the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + + Returns :const:`None` if the distribution is found but the installation + database records reporting the files associated with the distribuion package + are missing. + +.. class:: PackagePath + + A :class:`pathlib.PurePath` derived object with additional ``dist``, + ``size``, and ``hash`` properties corresponding to the distribution + package's installation metadata for that file. + +The :func:`!files` function takes a +`Distribution Package `_ +name and returns all of the files installed by this distribution. Each file is reported +as a :class:`PackagePath` instance. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -268,16 +331,16 @@ Once you have the file, you can also read its contents:: return s.encode('utf-8') return s -You can also use the ``locate`` method to get a the absolute path to the -file:: +You can also use the :meth:`!locate` method to get the absolute +path to the file:: >>> util.locate() # doctest: +SKIP PosixPath('/home/gustav/example/lib/site-packages/wheel/util.py') In the case where the metadata file listing files -(RECORD or SOURCES.txt) is missing, ``files()`` will -return ``None``. The caller may wish to wrap calls to -``files()`` in `always_iterable +(``RECORD`` or ``SOURCES.txt``) is missing, :func:`!files` will +return :const:`None`. The caller may wish to wrap calls to +:func:`!files` in `always_iterable `_ or otherwise guard against this condition if the target distribution is not known to have the metadata present. @@ -287,8 +350,16 @@ distribution is not known to have the metadata present. Distribution requirements ------------------------- +.. function:: requires(distribution_name) + + Return the declared dependency specifiers for the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + To get the full set of requirements for a `Distribution Package `_, -use the ``requires()`` +use the :func:`!requires` function:: >>> requires('wheel') # doctest: +SKIP @@ -301,6 +372,16 @@ function:: Mapping import to distribution packages --------------------------------------- +.. function:: packages_distributions() + + Return a mapping from the top level module and import package + names found via :attr:`sys.meta_path` to the names of the distribution + packages (if any) that provide the corresponding files. + + To allow for namespace packages (which may have members provided by + multiple distribution packages), each top level import name maps to a + list of distribution names rather than mapping directly to a single name. + A convenience method to resolve the `Distribution Package `_ name (or names, in the case of a namespace package) that provide each importable top-level @@ -320,23 +401,42 @@ function is not reliable with such installs. Distributions ============= -While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for -a Python `Distribution Package `_. You can -get the ``Distribution`` instance:: +.. function:: distribution(distribution_name) + + Return a :class:`Distribution` instance describing the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +.. class:: Distribution + + Details of an installed distribution package. + + Note: different :class:`!Distribution` instances do not currently compare + equal, even if they relate to the same installed distribution and + accordingly have the same attributes. + +While the module level API described above is the most common and convenient usage, +you can get all of that information from the :class:`!Distribution` class. +:class:`!Distribution` is an abstract object that represents the metadata for +a Python `Distribution Package `_. +You can get the concreate :class:`!Distribution` subclass instance for an installed +distribution package by calling the :func:`distribution` function:: >>> from importlib.metadata import distribution # doctest: +SKIP >>> dist = distribution('wheel') # doctest: +SKIP + >>> type(dist) # doctest: +SKIP + Thus, an alternative way to get the version number is through the -``Distribution`` instance:: +:class:`!Distribution` instance:: >>> dist.version # doctest: +SKIP '0.32.3' -There are all kinds of additional metadata available on the ``Distribution`` -instance:: +There are all kinds of additional metadata available on :class:`!Distribution` +instances:: >>> dist.metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' @@ -350,7 +450,7 @@ metadata:: 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' The full set of available metadata is not described here. -See the `Core metadata specifications `_ for additional details. +See the PyPA `Core metadata specification `_ for additional details. .. versionadded:: 3.13 The ``.origin`` property was added. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index e4cef1f3e3b7c0..9e088a598a6c08 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -249,7 +249,7 @@ ABC hierarchy:: An abstract method for finding a :term:`spec ` for the specified module. If this is a top-level import, *path* will be ``None``. Otherwise, this is a search for a subpackage or - module and *path* will be the value of :attr:`__path__` from the + module and *path* will be the value of :attr:`~module.__path__` from the parent package. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may use to make a more educated guess about what spec to return. @@ -355,34 +355,12 @@ ABC hierarchy:: (note that some of these attributes can change when a module is reloaded): - - :attr:`__name__` - The module's fully qualified name. - It is ``'__main__'`` for an executed module. - - - :attr:`__file__` - The location the :term:`loader` used to load the module. - For example, for modules loaded from a .py file this is the filename. - It is not set on all modules (e.g. built-in modules). - - - :attr:`__cached__` - The filename of a compiled version of the module's code. - It is not set on all modules (e.g. built-in modules). - - - :attr:`__path__` - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The import system passes this attribute to ``__import__()`` and to finders - in the same way as :data:`sys.path` but just for the package. - It is not set on non-package modules so it can be used - as an indicator that the module is a package. - - - :attr:`__package__` - The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`__name__`. - - - :attr:`__loader__` - The :term:`loader` used to load the module. + - :attr:`module.__name__` + - :attr:`module.__file__` + - :attr:`module.__cached__` *(deprecated)* + - :attr:`module.__path__` + - :attr:`module.__package__` *(deprecated)* + - :attr:`module.__loader__` *(deprecated)* When :meth:`exec_module` is available then backwards-compatible functionality is provided. @@ -418,7 +396,8 @@ ABC hierarchy:: can implement this abstract method to give direct access to the data stored. :exc:`OSError` is to be raised if the *path* cannot be found. The *path* is expected to be constructed using a module's - :attr:`__file__` attribute or an item from a package's :attr:`__path__`. + :attr:`~module.__file__` attribute or an item from a package's + :attr:`~module.__path__`. .. versionchanged:: 3.4 Raises :exc:`OSError` instead of :exc:`NotImplementedError`. @@ -505,9 +484,9 @@ ABC hierarchy:: .. abstractmethod:: get_filename(fullname) - An abstract method that is to return the value of :attr:`__file__` for - the specified module. If no path is available, :exc:`ImportError` is - raised. + An abstract method that is to return the value of + :attr:`~module.__file__` for the specified module. If no path is + available, :exc:`ImportError` is raised. If source code is available, then the method should return the path to the source file, regardless of whether a bytecode was used to load the @@ -1166,79 +1145,74 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's :attr:`__spec__` attribute. In the - descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object, - e.g. ``module.__spec__.origin == module.__file__``. Note, however, that + typically exposed as the module's :attr:`~module.__spec__` attribute. Many + of these attributes are also available directly on a module: for example, + ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is - no synchronization between the two objects. For example, it is possible to update - the module's :attr:`__file__` at runtime and this will not be automatically - reflected in the module's :attr:`__spec__.origin`, and vice versa. + no synchronization between the two objects. For example, it is possible to + update the module's :attr:`~module.__file__` at runtime and this will not be + automatically reflected in the module's + :attr:`__spec__.origin `, and vice versa. .. versionadded:: 3.4 .. attribute:: name - (:attr:`__name__`) - - The module's fully qualified name. - The :term:`finder` should always set this attribute to a non-empty string. + The module's fully qualified name (see :attr:`module.__name__`). + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (:attr:`__loader__`) - - The :term:`loader` used to load the module. - The :term:`finder` should always set this attribute. + The :term:`loader` used to load the module (see :attr:`module.__loader__`). + The :term:`finder` should always set this attribute. .. attribute:: origin - (:attr:`__file__`) - - The location the :term:`loader` should use to load the module. - For example, for modules loaded from a .py file this is the filename. - The :term:`finder` should always set this attribute to a meaningful value - for the :term:`loader` to use. In the uncommon case that there is not one - (like for namespace packages), it should be set to ``None``. + The location the :term:`loader` should use to load the module + (see :attr:`module.__file__`). + For example, for modules loaded from a ``.py`` file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (:attr:`__path__`) + A (possibly empty) :term:`sequence` of strings enumerating the locations + in which a package's submodules will be found + (see :attr:`module.__path__`). Most of the time there will only be a + single directory in this list. - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The :term:`finder` should set this attribute to a list, even an empty one, to indicate - to the import system that the module is a package. It should be set to ``None`` for - non-package modules. It is set automatically later to a special object for - namespace packages. + The :term:`finder` should set this attribute to a sequence, even an empty + one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - The :term:`finder` may set this attribute to an object containing additional, - module-specific data to use when loading the module. Otherwise it should be - set to ``None``. + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (:attr:`__cached__`) - - The filename of a compiled version of the module's code. - The :term:`finder` should always set this attribute but it may be ``None`` - for modules that do not need compiled code stored. + The filename of a compiled version of the module's code + (see :attr:`module.__cached__`). + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (:attr:`__package__`) - - (Read-only) The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`name`. + (Read-only) The fully qualified name of the package the module is in (or the + empty string for a top-level module). + See :attr:`module.__package__`. + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`!origin` is interpreted + and how the module's :attr:`~module.__file__` is populated. .. class:: AppleFrameworkLoader(name, path) @@ -1423,8 +1397,8 @@ an :term:`importer`. .. versionchanged:: 3.7 Raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if - **package** is in fact not a package (i.e. lacks a :attr:`__path__` - attribute). + **package** is in fact not a package (i.e. lacks a + :attr:`~module.__path__` attribute). .. function:: module_from_spec(spec) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 57e5cf7ae023d1..1eaf1cc5d9a68e 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -545,7 +545,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` method or a :meth:`~object.__delete__` method. Beyond that, the set of attributes varies. A :attr:`~definition.__name__` attribute is usually - sensible, and :attr:`!__doc__` often is. + sensible, and :attr:`~definition.__doc__` often is. Methods implemented via descriptors that also pass one of the other tests return ``False`` from the :func:`ismethoddescriptor` test, simply because the @@ -694,7 +694,7 @@ and its return annotation. To retrieve a :class:`!Signature` object, use the :func:`!signature` function. -.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) +.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, annotation_format=Format.VALUE) Return a :class:`Signature` object for the given *callable*: @@ -725,7 +725,12 @@ function. *globals*, *locals*, and *eval_str* parameters are passed into :func:`!annotationlib.get_annotations` when resolving the annotations; see the documentation for :func:`!annotationlib.get_annotations` - for instructions on how to use these parameters. + for instructions on how to use these parameters. A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, @@ -733,7 +738,7 @@ function. the ``eval()`` call(s) to un-stringize the annotations in :func:`annotationlib.get_annotations` could potentially raise any kind of exception. - A slash(/) in the signature of a function denotes that the parameters prior + A slash (/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see :ref:`the FAQ entry on positional-only parameters `. @@ -746,6 +751,9 @@ function. .. versionchanged:: 3.10 The *globals*, *locals*, and *eval_str* parameters were added. + .. versionchanged:: 3.14 + The *annotation_format* parameter was added. + .. note:: Some callables may not be introspectable in certain implementations of @@ -838,7 +846,7 @@ function. :class:`Signature` objects are also supported by the generic function :func:`copy.replace`. - .. method:: format(*, max_width=None) + .. method:: format(*, max_width=None, quote_annotation_strings=True) Create a string representation of the :class:`Signature` object. @@ -847,8 +855,17 @@ function. If the signature is longer than *max_width*, all parameters will be on separate lines. + If *quote_annotation_strings* is False, :term:`annotations ` + in the signature are displayed without opening and closing quotation + marks if they are strings. This is useful if the signature was created with the + :attr:`~annotationlib.Format.STRING` format or if + ``from __future__ import annotations`` was used. + .. versionadded:: 3.13 + .. versionchanged:: 3.14 + The *unquote_annotations* parameter was added. + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` (or its subclass) object for a given callable diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 508c20f4df6f5e..c138e903fa5a0f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -58,7 +58,7 @@ Iterator Arguments Results :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','ABC'], len) → (1, A B) (3, ABC)`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` @@ -93,7 +93,7 @@ Examples Results Itertool Functions ------------------ -The following module functions all construct and return iterators. Some provide +The following functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. @@ -131,11 +131,12 @@ loops that truncate the stream. total = function(total, element) yield total - The *function* argument can be set to :func:`min` for a running - minimum, :func:`max` for a running maximum, or :func:`operator.mul` - for a running product. `Amortization tables - `_ - can be built by accumulating interest and applying payments: + To compute a running minimum, set *function* to :func:`min`. + For a running maximum, set *function* to :func:`max`. + Or for a running product, set *function* to :func:`operator.mul`. + To build an `amortization table + `_, + accumulate the interest and apply payments: .. doctest:: @@ -202,10 +203,10 @@ loops that truncate the stream. .. function:: chain(*iterables) - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. This combines multiple data sources into a + single iterator. Roughly equivalent to:: def chain(*iterables): # chain('ABC', 'DEF') → A B C D E F @@ -353,10 +354,12 @@ loops that truncate the stream. def cycle(iterable): # cycle('ABCD') → A B C D A B C D A B C D ... + saved = [] for element in iterable: yield element saved.append(element) + while saved: for element in saved: yield element @@ -396,8 +399,10 @@ loops that truncate the stream. def filterfalse(predicate, iterable): # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 + if predicate is None: predicate = bool + for x in iterable: if not predicate(x): yield x @@ -474,7 +479,7 @@ loops that truncate the stream. If *start* is zero or ``None``, iteration starts at zero. Otherwise, elements from the iterable are skipped until *start* is reached. - If *stop* is ``None``, iteration continues until the iterable is + If *stop* is ``None``, iteration continues until the input is exhausted, if at all. Otherwise, it stops at the specified position. If *step* is ``None``, the step defaults to one. Elements are returned @@ -520,8 +525,10 @@ loops that truncate the stream. def pairwise(iterable): # pairwise('ABCDEFG') → AB BC CD DE EF FG + iterator = iter(iterable) a = next(iterator, None) + for b in iterator: yield a, b a = b @@ -584,7 +591,8 @@ loops that truncate the stream. .. function:: product(*iterables, repeat=1) - Cartesian product of input iterables. + `Cartesian product `_ + of the input iterables. Roughly equivalent to nested for-loops in a generator expression. For example, ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. @@ -691,25 +699,36 @@ loops that truncate the stream. def tee(iterable, n=2): if n < 0: - raise ValueError('n must be >= 0') - iterator = iter(iterable) - shared_link = [None, None] - return tuple(_tee(iterator, shared_link) for _ in range(n)) - - def _tee(iterator, link): - try: - while True: - if link[1] is None: - link[0] = next(iterator) - link[1] = [None, None] - value, link = link - yield value - except StopIteration: - return + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] - Once a :func:`tee` has been created, the original *iterable* should not be - used anywhere else; otherwise, the *iterable* could get advanced without - the tee objects being informed. + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value When the input *iterable* is already a tee iterator object, all members of the return tuple are constructed as if they had been @@ -717,6 +736,26 @@ loops that truncate the stream. allows nested :func:`tee` calls to share the same underlying data chain and to have a single update step rather than a chain of calls. + The flattening property makes tee iterators efficiently peekable: + + .. testcode:: + + def lookahead(tee_iterator): + "Return the next value without moving the input forward" + [forked_iterator] = tee(tee_iterator, 1) + return next(forked_iterator) + + .. doctest:: + + >>> iterator = iter('abcdef') + >>> [iterator] = tee(iterator, 1) # Make the input peekable + >>> next(iterator) # Move the iterator forward + 'a' + >>> lookahead(iterator) # Check next value + 'b' + >>> next(iterator) # Continue moving forward + 'b' + ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be raised when simultaneously using iterators returned by the same :func:`tee` call, even if the original *iterable* is threadsafe. @@ -933,15 +972,6 @@ and :term:`generators ` which incur interpreter overhead. iterators = cycle(islice(iterators, num_active)) yield from map(next, iterators) - def partition(predicate, iterable): - """Partition entries into false entries and true entries. - - If *predicate* is slow, consider wrapping it with functools.lru_cache(). - """ - # partition(is_odd, range(10)) → 0 2 4 6 8 and 1 3 5 7 9 - t1, t2 = tee(iterable) - return filterfalse(predicate, t1), filter(predicate, t2) - def subslices(seq): "Return all contiguous non-empty subslices of a sequence." # subslices('ABCD') → A AB ABC ABCD B BC BCD C CD D @@ -1159,15 +1189,19 @@ The following recipes have a more mathematical flavor: >>> list(it) ['d', 'e', 'f'] + >>> list(prepend(1, [2, 3, 4])) [1, 2, 3, 4] + >>> list(enumerate('abc')) [(0, 'a'), (1, 'b'), (2, 'c')] + >>> list(islice(tabulate(lambda x: 2*x), 4)) [0, 2, 4, 6] + >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] >>> # Verify the input is consumed greedily @@ -1176,6 +1210,7 @@ The following recipes have a more mathematical flavor: >>> list(input_iterator) [] + >>> it = iter(range(10)) >>> consume(it, 3) >>> # Verify the input is consumed lazily @@ -1186,6 +1221,7 @@ The following recipes have a more mathematical flavor: >>> next(it, 'Done') 'Done' + >>> nth('abcde', 3) 'd' >>> nth('abcde', 9) is None @@ -1197,6 +1233,7 @@ The following recipes have a more mathematical flavor: >>> list(it) ['d', 'e'] + >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] @@ -1210,24 +1247,19 @@ The following recipes have a more mathematical flavor: >>> ''.join(it) 'bbccc' + >>> quantify(range(99), lambda x: x%2==0) 50 - >>> quantify([True, False, False, True, True]) 3 - >>> quantify(range(12), predicate=lambda x: x%2==1) 6 + >>> a = [[1, 2, 3], [4, 5, 6]] >>> list(flatten(a)) [1, 2, 3, 4, 5, 6] - >>> list(repeatfunc(pow, 5, 2, 3)) - [8, 8, 8, 8, 8] - - >>> take(5, map(int, repeatfunc(random.random))) - [0, 0, 0, 0, 0] >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] @@ -1237,9 +1269,11 @@ The following recipes have a more mathematical flavor: >>> list(input_iterator) [] + >>> sum_of_squares([10, 20, 30]) 1400 + >>> list(reshape([(0, 1), (2, 3), (4, 5)], 3)) [(0, 1, 2), (3, 4, 5)] >>> M = [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] @@ -1260,6 +1294,7 @@ The following recipes have a more mathematical flavor: >>> list(reshape(M, 12)) [(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)] + >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] >>> # Verify that the inputs are consumed lazily @@ -1271,11 +1306,13 @@ The following recipes have a more mathematical flavor: >>> list(zip(input1, input2)) [(2, 22), (3, 33)] + >>> list(matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]])) [(49, 80), (41, 60)] >>> list(matmul([[2, 5], [7, 9], [3, 4]], [[7, 11, 5, 4, 9], [3, 5, 2, 6, 3]])) [(29, 47, 20, 38, 33), (76, 122, 53, 82, 90), (33, 53, 23, 36, 39)] + >>> list(convolve([1, -1, -20], [1, -3])) == [1, -4, -17, 60] True >>> data = [20, 40, 24, 32, 20, 28, 16] @@ -1298,6 +1335,7 @@ The following recipes have a more mathematical flavor: >>> list(signal_iterator) [30, 40, 50] + >>> from fractions import Fraction >>> from decimal import Decimal >>> polynomial_eval([1, -4, -17, 60], x=5) @@ -1329,6 +1367,7 @@ The following recipes have a more mathematical flavor: >>> polynomial_eval([11, 2], 7) == 11 * 7 + 2 True + >>> polynomial_from_roots([5, -4, 3]) [1, -4, -17, 60] >>> factored = lambda x: (x - 5) * (x + 4) * (x - 3) @@ -1336,9 +1375,11 @@ The following recipes have a more mathematical flavor: >>> all(factored(x) == expanded(x) for x in range(-10, 11)) True + >>> polynomial_derivative([1, -4, -17, 60]) [3, -8, -17] + >>> list(iter_index('AABCADEAF', 'A')) [0, 1, 4, 7] >>> list(iter_index('AABCADEAF', 'B')) @@ -1396,12 +1437,14 @@ The following recipes have a more mathematical flavor: >>> ''.join(input_iterator) 'DEAF' + >>> # Verify that the target value can be a sequence. >>> seq = [[10, 20], [30, 40], 30, 40, [30, 40], 50] >>> target = [30, 40] >>> list(iter_index(seq, target)) [1, 4] + >>> # Verify faithfulness to type specific index() method behaviors. >>> # For example, bytes and str perform continuous-subsequence searches >>> # that do not match the general behavior specified @@ -1431,6 +1474,7 @@ The following recipes have a more mathematical flavor: >>> set(sieve(10_000)).isdisjoint(carmichael) True + >>> list(factor(99)) # Code example 1 [3, 3, 11] >>> list(factor(1_000_000_000_000_007)) # Code example 2 @@ -1476,6 +1520,7 @@ The following recipes have a more mathematical flavor: >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True + >>> totient(0) # https://www.wolframalpha.com/input?i=totient+0 0 >>> first_totients = [1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, @@ -1495,9 +1540,15 @@ The following recipes have a more mathematical flavor: >>> totient(6 ** 20) == 1 * 2**19 * 2 * 3**19 # repeated primes True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] + + >>> list(repeatfunc(pow, 5, 2, 3)) + [8, 8, 8, 8, 8] + >>> take(5, map(int, repeatfunc(random.random))) + [0, 0, 0, 0, 0] >>> random.seed(85753098575309) >>> list(repeatfunc(random.random, 3)) [0.16370491282496968, 0.45889608687313455, 0.3747076837820118] @@ -1506,9 +1557,11 @@ The following recipes have a more mathematical flavor: >>> list(repeatfunc(pow, 3, 2, 5)) [32, 32, 32] + >>> list(grouper('abcdefg', 3, fillvalue='x')) [('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')] + >>> it = grouper('abcdefg', 3, incomplete='strict') >>> next(it) ('a', 'b', 'c') @@ -1522,6 +1575,7 @@ The following recipes have a more mathematical flavor: >>> list(grouper('abcdefg', n=3, incomplete='ignore')) [('a', 'b', 'c'), ('d', 'e', 'f')] + >>> list(sliding_window('ABCDEFG', 1)) [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] >>> list(sliding_window('ABCDEFG', 2)) @@ -1551,6 +1605,7 @@ The following recipes have a more mathematical flavor: ... 'zero or negative n not supported' + >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] @@ -1564,38 +1619,19 @@ The following recipes have a more mathematical flavor: >>> ''.join(chain(*input_iterators)) 'dijkopqr' - >>> def is_odd(x): - ... return x % 2 == 1 - - >>> evens, odds = partition(is_odd, range(10)) - >>> list(evens) - [0, 2, 4, 6, 8] - >>> list(odds) - [1, 3, 5, 7, 9] - >>> # Verify that the input is consumed lazily - >>> input_iterator = iter(range(10)) - >>> evens, odds = partition(is_odd, input_iterator) - >>> next(odds) - 1 - >>> next(odds) - 3 - >>> next(evens) - 0 - >>> list(input_iterator) - [4, 5, 6, 7, 8, 9] >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) True - >>> list(powerset('abcde')) == sorted(sorted(set(powerset('abcde'))), key=len) True + >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] >>> list(unique_everseen('ABBCcAD', str.casefold)) @@ -1610,6 +1646,7 @@ The following recipes have a more mathematical flavor: >>> ''.join(input_iterator) 'AAABBBCCDAABBB' + >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] >>> list(unique_justseen('ABBCcAD', str.casefold)) @@ -1624,6 +1661,7 @@ The following recipes have a more mathematical flavor: >>> ''.join(input_iterator) 'AAABBBCCDAABBB' + >>> list(unique([[1, 2], [3, 4], [1, 2]])) [[1, 2], [3, 4]] >>> list(unique('ABBcCAD', str.casefold)) @@ -1631,6 +1669,7 @@ The following recipes have a more mathematical flavor: >>> list(unique('ABBcCAD', str.casefold, reverse=True)) ['D', 'c', 'B', 'A'] + >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) >>> d['d'] = 4 @@ -1648,6 +1687,7 @@ The following recipes have a more mathematical flavor: >>> next(it, 'empty') 'empty' + >>> first_true('ABC0DEF1', '9', str.isdigit) '0' >>> # Verify that inputs are consumed lazily @@ -1728,21 +1768,36 @@ The following recipes have a more mathematical flavor: return true_iterator(), chain(transition, it) + def partition(predicate, iterable): + """Partition entries into false entries and true entries. + + If *predicate* is slow, consider wrapping it with functools.lru_cache(). + """ + # partition(is_odd, range(10)) → 0 2 4 6 8 and 1 3 5 7 9 + t1, t2 = tee(iterable) + return filterfalse(predicate, t1), filter(predicate, t2) + + + .. doctest:: :hide: >>> dotproduct([1,2,3], [4,5,6]) 32 + >>> sumprod([1,2,3], [4,5,6]) 32 + >>> list(islice(pad_none('abc'), 0, 6)) ['a', 'b', 'c', None, None, None] + >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] + >>> population = 'ABCDEFGH' >>> for r in range(len(population) + 1): ... seq = list(combinations(population, r)) @@ -1750,16 +1805,38 @@ The following recipes have a more mathematical flavor: ... assert nth_combination(population, r, i) == seq[i] ... for i in range(-len(seq), 0): ... assert nth_combination(population, r, i) == seq[i] - + ... >>> iterable = 'abcde' >>> r = 3 >>> combos = list(combinations(iterable, r)) >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) True + >>> it = iter('ABCdEfGhI') >>> all_upper, remainder = before_and_after(str.isupper, it) >>> ''.join(all_upper) 'ABC' >>> ''.join(remainder) 'dEfGhI' + + + >>> def is_odd(x): + ... return x % 2 == 1 + ... + >>> evens, odds = partition(is_odd, range(10)) + >>> list(evens) + [0, 2, 4, 6, 8] + >>> list(odds) + [1, 3, 5, 7, 9] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter(range(10)) + >>> evens, odds = partition(is_odd, input_iterator) + >>> next(odds) + 1 + >>> next(odds) + 3 + >>> next(evens) + 0 + >>> list(input_iterator) + [4, 5, 6, 7, 8, 9] diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 0246f99157024a..5f3c4840b5cc70 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -158,7 +158,8 @@ The :mod:`locale` module defines the following exception and functions: .. function:: nl_langinfo(option) - Return some locale-specific information as a string. This function is not + Return some locale-specific information as a string (or a tuple for + ``ALT_DIGITS``). This function is not available on all systems, and the set of possible options might also vary across platforms. The possible argument values are numbers, for which symbolic constants are available in the locale module. @@ -311,8 +312,16 @@ The :mod:`locale` module defines the following exception and functions: .. data:: ALT_DIGITS - Get a representation of up to 100 values used to represent the values - 0 to 99. + Get a tuple of up to 100 strings used to represent the values 0 to 99. + + The function temporarily sets the ``LC_CTYPE`` locale to the locale + of the category that determines the requested value (``LC_TIME``, + ``LC_NUMERIC``, ``LC_MONETARY`` or ``LC_MESSAGES``) if locales are + different and the resulting string is non-ASCII. + This temporary change affects other threads. + + .. versionchanged:: 3.14 + The function now temporarily sets the ``LC_CTYPE`` locale in some cases. .. function:: getdefaultlocale([envvars]) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 317ca8728248c8..0e9dc33ae2123a 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -753,16 +753,17 @@ The ``queue`` and ``listener`` keys are optional. If the ``queue`` key is present, the corresponding value can be one of the following: -* An object implementing the :class:`queue.Queue` public API. For instance, - this may be an actual instance of :class:`queue.Queue` or a subclass thereof, - or a proxy obtained by :meth:`multiprocessing.managers.SyncManager.Queue`. +* An object implementing the :meth:`Queue.put_nowait ` + and :meth:`Queue.get ` public API. For instance, this may be + an actual instance of :class:`queue.Queue` or a subclass thereof, or a proxy + obtained by :meth:`multiprocessing.managers.SyncManager.Queue`. This is of course only possible if you are constructing or modifying the configuration dictionary in code. * A string that resolves to a callable which, when called with no arguments, returns - the :class:`queue.Queue` instance to use. That callable could be a - :class:`queue.Queue` subclass or a function which returns a suitable queue instance, + the queue instance to use. That callable could be a :class:`queue.Queue` subclass + or a function which returns a suitable queue instance, such as ``my.module.queue_factory()``. * A dict with a ``'()'`` key which is constructed in the usual way as discussed in diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 6a67d6c75374af..235bcc281ac8f8 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -304,7 +304,8 @@ in a module, ``__name__`` is the module's name in the Python package namespace. parameter mirrors the equivalent one in the :mod:`warnings` module. The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` + dictionary which is used to populate the :attr:`~object.__dict__` of the + :class:`LogRecord` created for the logging event with user-defined attributes. These custom attributes can then be used as you like. For example, they could be incorporated into logged messages. For example:: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 80d6e4dae24463..036b8f44b9ff3b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -124,11 +124,11 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX except macOS. + Available on POSIX systems. - .. note:: - The default start method will change away from *fork* in Python 3.14. - Code that requires *fork* should explicitly specify that via + .. versionchanged:: 3.14 + This is no longer the default start method on any platform. + Code that requires *fork* must explicitly specify that via :func:`get_context` or :func:`set_start_method`. .. versionchanged:: 3.12 @@ -146,9 +146,11 @@ to start a process. These *start methods* are side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on POSIX platforms which support passing file descriptors - over Unix pipes such as Linux. + Available on POSIX platforms which support passing file descriptors over + Unix pipes such as Linux. The default on those. + .. versionchanged:: 3.14 + This became the default start method on POSIX platforms. .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for @@ -162,6 +164,13 @@ to start a process. These *start methods* are method should be considered unsafe as it can lead to crashes of the subprocess as macOS system libraries may start threads. See :issue:`33725`. +.. versionchanged:: 3.14 + + On POSIX platforms the default start method was changed from *fork* to + *forkserver* to retain the performance but avoid common multithreaded + process incompatibilities. See :gh:`84559`. + + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or diff --git a/Doc/library/os.rst b/Doc/library/os.rst index cd7ae7bdd7385a..33dd58febd9a5e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4602,7 +4602,7 @@ written in Python, such as a mail server's external command delivery program. See the :manpage:`pidfd_open(2)` man page for more details. - .. availability:: Linux >= 5.3 + .. availability:: Linux >= 5.3, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 .. data:: PIDFD_NONBLOCK diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4380122eb1be7d..30d0d385d0539c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1563,7 +1563,7 @@ Copying, moving and deleting This argument has no effect when copying files on Windows (where metadata is always preserved). - .. versionadded:: 3.14 + .. versionadded:: next .. method:: Path.copy_into(target_dir, *, follow_symlinks=True, \ @@ -1574,7 +1574,7 @@ Copying, moving and deleting :meth:`Path.copy`. Returns a new :class:`!Path` instance pointing to the copy. - .. versionadded:: 3.14 + .. versionadded:: next .. method:: Path.rename(target) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index d696161876e99d..6c099b22b38c21 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -159,12 +159,15 @@ slightly different way: is entered. -.. function:: set_trace(*, header=None) +.. function:: set_trace(*, header=None, commands=None) Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console just before debugging begins. + The *commands* argument, if given, is a list of commands to execute + when the debugger starts. + .. versionchanged:: 3.7 The keyword-only argument *header*. @@ -173,6 +176,9 @@ slightly different way: :func:`set_trace` will enter the debugger immediately, rather than on the next line of code to be executed. + .. versionadded:: 3.14 + The *commands* argument. + .. function:: post_mortem(traceback=None) Enter post-mortem debugging of the given *traceback* object. If no @@ -192,7 +198,7 @@ The ``run*`` functions and :func:`set_trace` are aliases for instantiating the access further features, you have to do this yourself: .. class:: Pdb(completekey='tab', stdin=None, stdout=None, skip=None, \ - nosigint=False, readrc=True) + nosigint=False, readrc=True, mode=None) :class:`Pdb` is the debugger class. @@ -211,6 +217,13 @@ access further features, you have to do this yourself: The *readrc* argument defaults to true and controls whether Pdb will load .pdbrc files from the filesystem. + The *mode* argument specifies how the debugger was invoked. + It impacts the workings of some debugger commands. + Valid values are ``'inline'`` (used by the breakpoint() builtin), + ``'cli'`` (used by the command line invocation) + or ``None`` (for backwards compatible behaviour, as before the *mode* + argument was added). + Example call to enable tracing with *skip*:: import pdb; pdb.Pdb(skip=['django.*']).set_trace() @@ -227,6 +240,9 @@ access further features, you have to do this yourself: .. versionchanged:: 3.6 The *readrc* argument. + .. versionadded:: 3.14 + Added the *mode* argument. + .. method:: run(statement, globals=None, locals=None) runeval(expression, globals=None, locals=None) runcall(function, *args, **kwds) @@ -423,17 +439,20 @@ can be overridden by the local file. Specifying any command resuming execution (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, - :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) + :pdbcmd:`return`, :pdbcmd:`until`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint—which could have its own command list, leading to ambiguities about which list to execute. - If you use the ``silent`` command in the command list, the usual message about - stopping at a breakpoint is not printed. This may be desirable for breakpoints - that are to print a specific message and then continue. If none of the other - commands print anything, you see no sign that the breakpoint was reached. + If the list of commands contains the ``silent`` command, or a command that + resumes execution, then the breakpoint message containing information about + the frame is not displayed. + + .. versionchanged:: 3.14 + Frame information will not be displayed if a command that resumes execution + is present in the command list. .. pdbcommand:: s(tep) @@ -669,6 +688,10 @@ can be overridden by the local file. History, breakpoints, actions and debugger options are preserved. :pdbcmd:`restart` is an alias for :pdbcmd:`run`. + .. versionchanged:: 3.14 + :pdbcmd:`run` and :pdbcmd:`restart` commands are disabled when the + debugger is invoked in ``'inline'`` mode. + .. pdbcommand:: q(uit) Quit from the debugger. The program being executed is aborted. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index f095cc84173737..4a39d53a5f1440 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -26,7 +26,8 @@ support. __path__ = extend_path(__path__, __name__) For each directory on :data:`sys.path` that has a subdirectory that matches the - package name, add the subdirectory to the package's :attr:`__path__`. This is useful + package name, add the subdirectory to the package's + :attr:`~module.__path__`. This is useful if one wants to distribute different parts of a single logical package as multiple directories. diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index f7ca1e045699eb..70e9c604ebac4f 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -21,7 +21,7 @@ modules. The documentation can be presented as pages of text on the console, served to a web browser, or saved to HTML files. For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`!__doc__` attribute) of the object, +derived from the docstring (i.e. the :attr:`~definition.__doc__` attribute) of the object, and recursively of its documentable members. If there is no docstring, :mod:`!pydoc` tries to obtain a description from the block of comment lines just above the definition of the class, function or method in the source file, or at diff --git a/Doc/library/random.rst b/Doc/library/random.rst index c7f6b0bdd5b822..ef0cfb0e76cef6 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -741,7 +741,7 @@ The following options are accepted: .. option:: -f --float - Print a random floating-point number between 1 and N inclusive, + Print a random floating-point number between 0 and N inclusive, using :meth:`uniform`. If no options are given, the output depends on the input: diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index e623c3df7beba6..d25701c087ed07 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -449,9 +449,10 @@ Directory and files operations *mode* is a permission mask passed to :func:`os.access`, by default determining if the file exists and is executable. - *path* is a "``PATH`` string" specifying the lookup directory list. When no - *path* is specified, the results of :func:`os.environ` are used, returning - either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the directories to look in, + delimited by :data:`os.pathsep`. When no *path* is specified, the + :envvar:`PATH` environment variable is read from :data:`os.environ`, + falling back to :data:`os.defpath` if it is not set. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the @@ -460,9 +461,9 @@ Directory and files operations consulting the current working directory for executables: set the environment variable ``NoDefaultCurrentDirectoryInExePath``. - Also on Windows, the ``PATHEXT`` variable is used to resolve commands - that may not already include an extension. For example, if you call - ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` + Also on Windows, the :envvar:`PATHEXT` environment variable is used to + resolve commands that may not already include an extension. For example, + if you call ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* directories. For example, on Windows:: diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 79c4948e99e967..17fcb2b3707978 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -411,7 +411,7 @@ The :mod:`signal` module defines the following functions: See the :manpage:`pidfd_send_signal(2)` man page for more information. - .. availability:: Linux >= 5.1 + .. availability:: Linux >= 5.1, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 714507ce73c807..a6e2e3b8928ebe 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4505,14 +4505,14 @@ can be used interchangeably to index the same dictionary entry. ``dict([('foo', 100), ('bar', 200)])``, ``dict(foo=100, bar=200)`` If no positional argument is given, an empty dictionary is created. - If a positional argument is given and it is a mapping object, a dictionary - is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterable` object. Each item in - the iterable must itself be an iterable with exactly two objects. The - first object of each item becomes a key in the new dictionary, and the - second object the corresponding value. If a key occurs more than once, the - last value for that key becomes the corresponding value in the new - dictionary. + If a positional argument is given and it defines a ``keys()`` method, a + dictionary is created by calling :meth:`~object.__getitem__` on the argument with + each returned key from the method. Otherwise, the positional argument must be an + :term:`iterable` object. Each item in the iterable must itself be an iterable + with exactly two elements. The first element of each item becomes a key in the + new dictionary, and the second element the corresponding value. If a key occurs + more than once, the last value for that key becomes the corresponding value in + the new dictionary. If keyword arguments are given, the keyword arguments and their values are added to the dictionary created from the positional argument. If a key @@ -4669,10 +4669,11 @@ can be used interchangeably to index the same dictionary entry. Update the dictionary with the key/value pairs from *other*, overwriting existing keys. Return ``None``. - :meth:`update` accepts either another dictionary object or an iterable of - key/value pairs (as tuples or other iterables of length two). If keyword - arguments are specified, the dictionary is then updated with those - key/value pairs: ``d.update(red=1, blue=2)``. + :meth:`update` accepts either another object with a ``keys()`` method (in + which case :meth:`~object.__getitem__` is called with every key returned from + the method). or an iterable of key/value pairs (as tuples or other iterables + of length two). If keyword arguments are specified, the dictionary is then + updated with those key/value pairs: ``d.update(red=1, blue=2)``. .. method:: values() @@ -5521,22 +5522,6 @@ types, where they are relevant. Some of these are not reported by the :func:`dir` built-in function. -.. attribute:: object.__dict__ - - A dictionary or other mapping object used to store an object's (writable) - attributes. - - -.. attribute:: instance.__class__ - - The class to which a class instance belongs. - - -.. attribute:: class.__bases__ - - The tuple of base classes of a class object. - - .. attribute:: definition.__name__ The name of the class, function, method, descriptor, or @@ -5551,43 +5536,24 @@ types, where they are relevant. Some of these are not reported by the .. versionadded:: 3.3 -.. attribute:: definition.__type_params__ +.. attribute:: definition.__module__ - The :ref:`type parameters ` of generic classes, functions, - and :ref:`type aliases `. + The name of the module in which a class or function was defined. - .. versionadded:: 3.12 - - -.. attribute:: class.__mro__ - - This attribute is a tuple of classes that are considered when looking for - base classes during method resolution. - - -.. method:: class.mro() - This method can be overridden by a metaclass to customize the method - resolution order for its instances. It is called at class instantiation, and - its result is stored in :attr:`~class.__mro__`. +.. attribute:: definition.__doc__ + The documentation string of a class or function, or ``None`` if undefined. -.. method:: class.__subclasses__ - - Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. The list is in - definition order. Example:: - - >>> int.__subclasses__() - [, , , ] +.. attribute:: definition.__type_params__ -.. attribute:: class.__static_attributes__ + The :ref:`type parameters ` of generic classes, functions, + and :ref:`type aliases `. For classes and functions that + are not generic, this will be an empty tuple. - A tuple containing names of attributes of this class which are accessed - through ``self.X`` from any function in its body. + .. versionadded:: 3.12 - .. versionadded:: 3.13 .. _int_max_str_digits: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1f316307965c11..49aeb28d57c8d1 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -509,9 +509,8 @@ The available presentation types for :class:`float` and | | significant digits. With no precision given, uses a | | | precision of ``6`` digits after the decimal point for | | | :class:`float`, and shows all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | + | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | + | | point is omitted unless the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -522,9 +521,8 @@ The available presentation types for :class:`float` and | | precision given, uses a precision of ``6`` digits after | | | the decimal point for :class:`float`, and uses a | | | precision large enough to show all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | + | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | + | | point is omitted unless the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | @@ -574,11 +572,13 @@ The available presentation types for :class:`float` and | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | + | None | For :class:`float` this is like the ``'g'`` type, except | | | that when fixed-point notation is used to format the | | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | + | | decimal point, and switches to the scientific notation | + | | when ``exp >= p - 1``. When the precision is not | + | | specified, the latter will be as large as needed to | + | | represent the given value faithfully. | | | | | | For :class:`~decimal.Decimal`, this is the same as | | | either ``'g'`` or ``'G'`` depending on the value of | diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 4769affdf1d666..3ea9e5ba071289 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -267,12 +267,26 @@ platform-dependent. | ``P`` | :c:expr:`void \*` | integer | | \(5) | +--------+--------------------------+--------------------+----------------+------------+ +Additionally, if IEC 60559 compatible complex arithmetic (Annex G of the +C11 standard) is supported, the following format characters are available: + ++--------+--------------------------+--------------------+----------------+------------+ +| Format | C Type | Python type | Standard size | Notes | ++========+==========================+====================+================+============+ +| ``E`` | :c:expr:`float complex` | complex | 8 | \(10) | ++--------+--------------------------+--------------------+----------------+------------+ +| ``C`` | :c:expr:`double complex` | complex | 16 | \(10) | ++--------+--------------------------+--------------------+----------------+------------+ + .. versionchanged:: 3.3 Added support for the ``'n'`` and ``'N'`` formats. .. versionchanged:: 3.6 Added support for the ``'e'`` format. +.. versionchanged:: 3.14 + Added support for the ``'E'`` and ``'C'`` formats. + Notes: @@ -349,6 +363,11 @@ Notes: of bytes. As a special case, ``'0s'`` means a single, empty string (while ``'0c'`` means 0 characters). +(10) + For the ``'E'`` and ``'C'`` format characters, the packed representation uses + the IEEE 754 binary32 and binary64 format for components of the complex + number, regardless of the floating-point format used by the platform. + A format character may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 8ebcb3bcf1b7b4..56cd6b8afaa73e 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -167,11 +167,12 @@ Examining Symbol Tables .. method:: get_nonlocals() - Return a tuple containing names of nonlocals in this function. + Return a tuple containing names of explicitly declared nonlocals in this function. .. method:: get_frees() - Return a tuple containing names of free variables in this function. + Return a tuple containing names of :term:`free (closure) variables ` + in this function. .. class:: Class @@ -255,7 +256,7 @@ Examining Symbol Tables Return ``True`` if the symbol is a type parameter. - .. versionadded:: 3.14 + .. versionadded:: next .. method:: is_global() @@ -302,7 +303,7 @@ Examining Symbol Tables be free from the perspective of ``C.method``, thereby allowing the latter to return *1* at runtime and not *2*. - .. versionadded:: 3.14 + .. versionadded:: next .. method:: is_assigned() @@ -312,13 +313,13 @@ Examining Symbol Tables Return ``True`` if the symbol is a comprehension iteration variable. - .. versionadded:: 3.14 + .. versionadded:: next .. method:: is_comp_cell() Return ``True`` if the symbol is a cell in an inlined comprehension. - .. versionadded:: 3.14 + .. versionadded:: next .. method:: is_namespace() diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index ac8bcceaca5aeb..f7140af2494898 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -50,16 +50,14 @@ Registering and using tools *tool_id* must be in the range 0 to 5 inclusive. Raises a :exc:`ValueError` if *tool_id* is in use. -.. function:: free_tool_id(tool_id: int, /) -> None +.. function:: clear_tool_id(tool_id: int, /) -> None - Should be called once a tool no longer requires *tool_id*. + Unregister all events and callback functions associated with *tool_id*. -.. note:: +.. function:: free_tool_id(tool_id: int, /) -> None - :func:`free_tool_id` will not disable global or local events associated - with *tool_id*, nor will it unregister any callback functions. This - function is only intended to be used to notify the VM that the - particular *tool_id* is no longer in use. + Should be called once a tool no longer requires *tool_id*. + Will call :func:`clear_tool_id` before releasing *tool_id*. .. function:: get_tool(tool_id: int, /) -> str | None diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b0e40a4ea06946..20a06a1ecd1a4c 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1274,7 +1274,8 @@ always available. that implement Python's default import semantics. The :meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at least the absolute name of the module being imported. If the module to be - imported is contained in a package, then the parent package's :attr:`__path__` + imported is contained in a package, then the parent package's + :attr:`~module.__path__` attribute is passed in as a second argument. The method returns a :term:`module spec`, or ``None`` if the module cannot be found. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 12f86043095598..04d28aee0f8672 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -946,7 +946,7 @@ The :mod:`test.support` module defines the following functions: other modules, possibly a C backend (like ``csv`` and its ``_csv``). The *extra* argument can be a set of names that wouldn't otherwise be automatically - detected as "public", like objects without a proper ``__module__`` + detected as "public", like objects without a proper :attr:`~definition.__module__` attribute. If provided, it will be added to the automatically detected ones. The *not_exported* argument can be a set of names that must not be treated diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 401e12be45f418..100a92b73d5497 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -8,11 +8,15 @@ -------------- -This module provides a standard interface to extract, format and print stack -traces of Python programs. It exactly mimics the behavior of the Python -interpreter when it prints a stack trace. This is useful when you want to print -stack traces under program control, such as in a "wrapper" around the -interpreter. +This module provides a standard interface to extract, format and print +stack traces of Python programs. It is more flexible than the +interpreter's default traceback display, and therefore makes it +possible to configure certain aspects of the output. Finally, +it contains a utility for capturing enough information about an +exception to print it later, without the need to save a reference +to the actual exception. Since exceptions can be the roots of large +objects graph, this utility can significantly improve +memory management. .. index:: pair: object; traceback @@ -29,7 +33,20 @@ which are assigned to the :attr:`~BaseException.__traceback__` field of Module :mod:`pdb` Interactive source code debugger for Python programs. -The module defines the following functions: +The module's API can be divided into two parts: + +* Module-level functions offering basic functionality, which are useful for interactive + inspection of exceptions and tracebacks. + +* :class:`TracebackException` class and its helper classes + :class:`StackSummary` and :class:`FrameSummary`. These offer both more + flexibility in the output generated and the ability to store the information + necessary for later formatting without holding references to actual exception + and traceback objects. + + +Module-Level Functions +---------------------- .. function:: print_tb(tb, limit=None, file=None) @@ -237,7 +254,6 @@ The module defines the following functions: .. versionadded:: 3.5 -The module also defines the following classes: :class:`!TracebackException` Objects ------------------------------------ @@ -245,12 +261,17 @@ The module also defines the following classes: .. versionadded:: 3.5 :class:`!TracebackException` objects are created from actual exceptions to -capture data for later printing in a lightweight fashion. +capture data for later printing. They offer a more lightweight method of +storing this information by avoiding holding references to +:ref:`traceback` and :ref:`frame` objects +In addition, they expose more options to configure the output compared to +the module-level functions described above. .. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. + Capture an exception for later rendering. The meaning of *limit*, + *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` + class. If *compact* is true, only data that is required by :class:`!TracebackException`'s :meth:`format` method @@ -509,8 +530,8 @@ in a :ref:`traceback `. .. _traceback-example: -Traceback Examples ------------------- +Examples of Using the Module-Level Functions +-------------------------------------------- This simple example implements a basic read-eval-print loop, similar to (but less useful than) the standard Python interactive interpreter loop. For a more @@ -549,8 +570,7 @@ exception and traceback: try: lumberjack() - except IndexError: - exc = sys.exception() + except IndexError as exc: print("*** print_tb:") traceback.print_tb(exc.__traceback__, limit=1, file=sys.stdout) print("*** print_exception:") @@ -653,5 +673,88 @@ This last example demonstrates the final few formatting functions: [' File "spam.py", line 3, in \n spam.eggs()\n', ' File "eggs.py", line 42, in eggs\n return "bacon"\n'] >>> an_error = IndexError('tuple index out of range') - >>> traceback.format_exception_only(type(an_error), an_error) + >>> traceback.format_exception_only(an_error) ['IndexError: tuple index out of range\n'] + + +Examples of Using :class:`TracebackException` +--------------------------------------------- + +With the helper class, we have more options:: + + >>> import sys + >>> from traceback import TracebackException + >>> + >>> def lumberjack(): + ... bright_side_of_life() + ... + >>> def bright_side_of_life(): + ... t = "bright", "side", "of", "life" + ... return t[5] + ... + >>> try: + ... lumberjack() + ... except IndexError as e: + ... exc = e + ... + >>> try: + ... try: + ... lumberjack() + ... except: + ... 1/0 + ... except Exception as e: + ... chained_exc = e + ... + >>> # limit works as with the module-level functions + >>> TracebackException.from_exception(exc, limit=-2).print() + Traceback (most recent call last): + File "", line 6, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 10, in bright_side_of_life + return t[5] + ~^^^ + IndexError: tuple index out of range + + >>> # capture_locals adds local variables in frames + >>> TracebackException.from_exception(exc, limit=-2, capture_locals=True).print() + Traceback (most recent call last): + File "", line 6, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 10, in bright_side_of_life + return t[5] + ~^^^ + t = ("bright", "side", "of", "life") + IndexError: tuple index out of range + + >>> # The *chain* kwarg to print() controls whether chained + >>> # exceptions are displayed + >>> TracebackException.from_exception(chained_exc).print() + Traceback (most recent call last): + File "", line 4, in + lumberjack() + ~~~~~~~~~~^^ + File "", line 7, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 11, in bright_side_of_life + return t[5] + ~^^^ + IndexError: tuple index out of range + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "", line 6, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> TracebackException.from_exception(chained_exc).print(chain=False) + Traceback (most recent call last): + File "", line 6, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index da801d4dc1f5b3..efa4b6f8f1d3f9 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -14,6 +14,11 @@ from turtle import * turtle = Turtle() +.. testcleanup:: + + import os + os.remove("my_drawing.ps") + -------------- Introduction diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 116868c24be864..439e119461f798 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -91,8 +91,8 @@ Dynamic Type Creation For classes that have an ``__orig_bases__`` attribute, this function returns the value of ``cls.__orig_bases__``. - For classes without the ``__orig_bases__`` attribute, ``cls.__bases__`` is - returned. + For classes without the ``__orig_bases__`` attribute, + :attr:`cls.__bases__ ` is returned. Examples:: @@ -199,7 +199,7 @@ Standard names are defined for the following types: .. data:: CellType The type for cell objects: such objects are used as containers for - a function's free variables. + a function's :term:`closure variables `. .. versionadded:: 3.8 @@ -260,63 +260,18 @@ Standard names are defined for the following types: The type of :term:`modules `. The constructor takes the name of the module to be created and optionally its :term:`docstring`. - .. note:: - Use :func:`importlib.util.module_from_spec` to create a new module if you - wish to set the various import-controlled attributes. - - .. attribute:: __doc__ - - The :term:`docstring` of the module. Defaults to ``None``. - - .. attribute:: __loader__ - - The :term:`loader` which loaded the module. Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__loader__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __name__ - - The name of the module. Expected to match - :attr:`importlib.machinery.ModuleSpec.name`. - - .. attribute:: __package__ - - Which :term:`package` a module belongs to. If the module is top-level - (i.e. not a part of any specific package) then the attribute should be set - to ``''``, else it should be set to the name of the package (which can be - :attr:`__name__` if the module is a package itself). Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__package__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __spec__ - - A record of the module's import-system-related state. Expected to be an - instance of :class:`importlib.machinery.ModuleSpec`. + .. seealso:: - .. versionadded:: 3.4 + :ref:`Documentation on module objects ` + Provides details on the special attributes that can be found on + instances of :class:`!ModuleType`. + :func:`importlib.util.module_from_spec` + Modules created using the :class:`!ModuleType` constructor are + created with many of their special attributes unset or set to default + values. :func:`!module_from_spec` provides a more robust way of + creating :class:`!ModuleType` instances which ensures the various + attributes are set appropriately. .. data:: EllipsisType @@ -392,7 +347,7 @@ Standard names are defined for the following types: In addition, when a class is defined with a :attr:`~object.__slots__` attribute, then for each slot, an instance of :class:`!MemberDescriptorType` will be added as an attribute - on the class. This allows the slot to appear in the class's :attr:`~object.__dict__`. + on the class. This allows the slot to appear in the class's :attr:`~type.__dict__`. .. impl-detail:: diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index f52c593a086c0a..cd8b90854b0e94 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2349,7 +2349,9 @@ types. Backward-compatible usage:: - # For creating a generic NamedTuple on Python 3.11 or lower + # For creating a generic NamedTuple on Python 3.11 + T = TypeVar("T") + class Group(NamedTuple, Generic[T]): key: T group: list[T] @@ -3269,7 +3271,8 @@ Introspection helpers empty dictionary is returned. * If *obj* is a class ``C``, the function returns a dictionary that merges annotations from ``C``'s base classes with those on ``C`` directly. This - is done by traversing ``C.__mro__`` and iteratively combining + is done by traversing :attr:`C.__mro__ ` and iteratively + combining ``__annotations__`` dictionaries. Annotations on classes appearing earlier in the :term:`method resolution order` always take precedence over annotations on classes appearing later in the method resolution order. @@ -3426,7 +3429,7 @@ Introspection helpers * Replaces type hints that evaluate to :const:`!None` with :class:`types.NoneType`. * Supports the :attr:`~annotationlib.Format.FORWARDREF` and - :attr:`~annotationlib.Format.SOURCE` formats. + :attr:`~annotationlib.Format.STRING` formats. *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. *owner*, if given, should be the object that holds the annotations that diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index e15f8a4e903dc5..eae3ef2888eae0 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -68,7 +68,7 @@ available, and then make assertions about how they have been used: 3 >>> thing.method.assert_called_with(3, 4, 5, key='value') -:attr:`side_effect` allows you to perform side effects, including raising an +:attr:`~Mock.side_effect` allows you to perform side effects, including raising an exception when a mock is called: >>> from unittest.mock import Mock @@ -239,7 +239,7 @@ the *new_callable* argument to :func:`patch`. Accessing any attribute not in this list will raise an :exc:`AttributeError`. If *spec* is an object (rather than a list of strings) then - :attr:`~instance.__class__` returns the class of the spec object. This + :attr:`~object.__class__` returns the class of the spec object. This allows mocks to pass :func:`isinstance` tests. * *spec_set*: A stricter variant of *spec*. If used, attempting to *set* @@ -401,6 +401,8 @@ the *new_callable* argument to :func:`patch`. The reset_mock method resets all the call attributes on a mock object: + .. doctest:: + >>> mock = Mock(return_value=None) >>> mock('hello') >>> mock.called @@ -409,20 +411,41 @@ the *new_callable* argument to :func:`patch`. >>> mock.called False - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the + reuse the same object. + + *return_value* parameter when set to ``True`` resets :attr:`return_value`: + + .. doctest:: + + >>> mock = Mock(return_value=5) + >>> mock('hello') + 5 + >>> mock.reset_mock(return_value=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + *side_effect* parameter when set to ``True`` resets :attr:`side_effect`: + + .. doctest:: + + >>> mock = Mock(side_effect=ValueError) + >>> mock('hello') + Traceback (most recent call last): + ... + ValueError + >>> mock.reset_mock(side_effect=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + Note that :meth:`reset_mock` *doesn't* clear the :attr:`return_value`, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - :attr:`return_value` or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. + set using normal assignment by default. - .. note:: *return_value*, and *side_effect* are keyword-only - arguments. + Child mocks are reset as well. + .. versionchanged:: 3.6 + Added two keyword-only arguments to the reset_mock function. .. method:: mock_add_spec(spec, spec_set=False) @@ -737,8 +760,8 @@ the *new_callable* argument to :func:`patch`. .. attribute:: __class__ - Normally the :attr:`__class__` attribute of an object will return its type. - For a mock object with a :attr:`spec`, ``__class__`` returns the spec class + Normally the :attr:`!__class__` attribute of an object will return its type. + For a mock object with a :attr:`!spec`, :attr:`!__class__` returns the spec class instead. This allows mock objects to pass :func:`isinstance` tests for the object they are replacing / masquerading as: @@ -746,7 +769,7 @@ the *new_callable* argument to :func:`patch`. >>> isinstance(mock, int) True - :attr:`__class__` is assignable to, this allows a mock to pass an + :attr:`!__class__` is assignable to, this allows a mock to pass an :func:`isinstance` check without forcing you to use a spec: >>> mock = Mock() @@ -760,8 +783,8 @@ the *new_callable* argument to :func:`patch`. meaning of :class:`Mock`, with the exception of *return_value* and *side_effect* which have no meaning on a non-callable mock. -Mock objects that use a class or an instance as a :attr:`spec` or -:attr:`spec_set` are able to pass :func:`isinstance` tests: +Mock objects that use a class or an instance as a :attr:`!spec` or +:attr:`!spec_set` are able to pass :func:`isinstance` tests: >>> mock = Mock(spec=SomeClass) >>> isinstance(mock, SomeClass) @@ -1175,7 +1198,7 @@ Calls made to the object will be recorded in the attributes like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`. If :attr:`~Mock.side_effect` is set then it will be called after the call has -been recorded, so if :attr:`side_effect` raises an exception the call is still +been recorded, so if :attr:`!side_effect` raises an exception the call is still recorded. The simplest way to make a mock raise an exception when called is to make @@ -1196,8 +1219,8 @@ The simplest way to make a mock raise an exception when called is to make >>> m.mock_calls [call(1, 2, 3), call('two', 'three', 'four')] -If :attr:`side_effect` is a function then whatever that function returns is what -calls to the mock return. The :attr:`side_effect` function is called with the +If :attr:`~Mock.side_effect` is a function then whatever that function returns is what +calls to the mock return. The :attr:`!side_effect` function is called with the same arguments as the mock. This allows you to vary the return value of the call dynamically, based on the input: @@ -1214,7 +1237,7 @@ call dynamically, based on the input: If you want the mock to still return the default return value (a new mock), or any set return value, then there are two ways of doing this. Either return -:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`: +:attr:`~Mock.return_value` from inside :attr:`~Mock.side_effect`, or return :data:`DEFAULT`: >>> m = MagicMock() >>> def side_effect(*args, **kwargs): @@ -1231,8 +1254,8 @@ any set return value, then there are two ways of doing this. Either return >>> m() 3 -To remove a :attr:`side_effect`, and return to the default behaviour, set the -:attr:`side_effect` to ``None``: +To remove a :attr:`~Mock.side_effect`, and return to the default behaviour, set the +:attr:`!side_effect` to ``None``: >>> m = MagicMock(return_value=6) >>> def side_effect(*args, **kwargs): @@ -1245,7 +1268,7 @@ To remove a :attr:`side_effect`, and return to the default behaviour, set the >>> m() 6 -The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock +The :attr:`~Mock.side_effect` can also be any iterable object. Repeated calls to the mock will return values from the iterable (until the iterable is exhausted and a :exc:`StopIteration` is raised): @@ -1286,7 +1309,7 @@ objects of any type. You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an :exc:`AttributeError` when an attribute is fetched. You can do this by providing -an object as a :attr:`spec` for a mock, but that isn't always convenient. +an object as a :attr:`!spec` for a mock, but that isn't always convenient. You "block" attributes by deleting them. Once deleted, accessing an attribute will raise an :exc:`AttributeError`. @@ -1455,7 +1478,7 @@ patch If you are patching builtins in a module then you don't need to pass ``create=True``, it will be added by default. - Patch can be used as a :class:`TestCase` class decorator. It works by + Patch can be used as a :class:`~unittest.TestCase` class decorator. It works by decorating each test method in the class. This reduces the boilerplate code when your test methods share a common patchings set. :func:`patch` finds tests by looking for method names that start with ``patch.TEST_PREFIX``. @@ -1493,7 +1516,7 @@ If the class is instantiated multiple times you could use can set the *return_value* to be anything you want. To configure return values on methods of *instances* on the patched class -you must do this on the :attr:`return_value`. For example:: +you must do this on the :attr:`~Mock.return_value`. For example:: >>> class Class: ... def method(self): @@ -1815,13 +1838,13 @@ context manager is a dictionary where created mocks are keyed by name:: patch methods: start and stop ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do +All the patchers have :meth:`!start` and :meth:`!stop` methods. These make it simpler to do patching in ``setUp`` methods or where you want to do multiple patches without nesting decorators or with statements. To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as normal and keep a reference to the returned ``patcher`` object. You can then -call :meth:`start` to put the patch in place and :meth:`stop` to undo it. +call :meth:`!start` to put the patch in place and :meth:`!stop` to undo it. If you are using :func:`patch` to create a mock for you then it will be returned by the call to ``patcher.start``. :: @@ -1838,7 +1861,7 @@ the call to ``patcher.start``. :: A typical use case for this might be for doing multiple patches in the ``setUp`` -method of a :class:`TestCase`:: +method of a :class:`~unittest.TestCase`:: >>> class MyTest(unittest.TestCase): ... def setUp(self): @@ -2511,7 +2534,7 @@ behaviour you can switch it off by setting the module level switch Alternatively you can just use ``vars(my_mock)`` (instance members) and ``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:const:`mock.FILTER_DIR`. +:const:`FILTER_DIR`. mock_open @@ -2526,7 +2549,7 @@ mock_open default) then a :class:`MagicMock` will be created for you, with the API limited to methods or attributes available on standard file handles. - *read_data* is a string for the :meth:`~io.IOBase.read`, + *read_data* is a string for the :meth:`~io.RawIOBase.read`, :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods of the file handle to return. Calls to those methods will take data from *read_data* until it is depleted. The mock of these methods is pretty @@ -2538,7 +2561,7 @@ mock_open .. versionchanged:: 3.4 Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. - The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather + The mock of :meth:`~io.RawIOBase.read` changed to consume *read_data* rather than returning it on each call. .. versionchanged:: 3.5 @@ -2590,7 +2613,7 @@ And for reading files:: Autospeccing ~~~~~~~~~~~~ -Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the +Autospeccing is based on the existing :attr:`!spec` feature of mock. It limits the api of mocks to the api of an original object (the spec), but it is recursive (implemented lazily) so that attributes of mocks only have the same api as the attributes of the spec. In addition mocked functions / methods have the @@ -2615,8 +2638,8 @@ unit tests. Testing everything in isolation is all fine and dandy, but if you don't test how your units are "wired together" there is still lots of room for bugs that tests might have caught. -:mod:`mock` already provides a feature to help with this, called speccing. If you -use a class or instance as the :attr:`spec` for a mock then you can only access +:mod:`unittest.mock` already provides a feature to help with this, called speccing. If you +use a class or instance as the :attr:`!spec` for a mock then you can only access attributes on the mock that exist on the real class: >>> from urllib import request @@ -2654,7 +2677,7 @@ Here's an example of it in use:: >>> mock_request.Request -You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two +You can see that :class:`!request.Request` has a spec. :class:`!request.Request` takes two arguments in the constructor (one of which is *self*). Here's what happens if we try to call it incorrectly:: @@ -2670,8 +2693,8 @@ specced mocks):: >>> req -:class:`Request` objects are not callable, so the return value of instantiating our -mocked out :class:`request.Request` is a non-callable mock. With the spec in place +:class:`!Request` objects are not callable, so the return value of instantiating our +mocked out :class:`!request.Request` is a non-callable mock. With the spec in place any typos in our asserts will raise the correct error:: >>> req.add_header('spam', 'eggs') @@ -2823,8 +2846,8 @@ Sealing mocks .. versionadded:: 3.7 -Order of precedence of :attr:`side_effect`, :attr:`return_value` and *wraps* ----------------------------------------------------------------------------- +Order of precedence of :attr:`!side_effect`, :attr:`!return_value` and *wraps* +------------------------------------------------------------------------------ The order of their precedence is: diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index cf6c5437be4fd1..5205c6c211d9bf 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -37,14 +37,14 @@ A virtual environment is (amongst other things): are by default isolated from software in other virtual environments and Python interpreters and libraries installed in the operating system. -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in +* Contained in a directory, conventionally named ``.venv`` or ``venv`` in the project directory, or under a container directory for lots of virtual environments, such as ``~/.virtualenvs``. * Not checked into source control systems such as Git. * Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment + scratch. You don't place any project code in the environment. * Not considered as movable or copyable -- you just recreate the same environment in the target location. @@ -61,7 +61,127 @@ See :pep:`405` for more background on Python virtual environments. Creating virtual environments ----------------------------- -.. include:: /using/venv-create.inc +:ref:`Virtual environments ` are created by executing the ``venv`` +module: + +.. code-block:: shell + + python -m venv /path/to/new/virtual/environment + +This creates the target directory (including parent directories as needed) +and places a :file:`pyvenv.cfg` file in it with a ``home`` key +pointing to the Python installation from which the command was run. +It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory +containing a copy or symlink of the Python executable +(as appropriate for the platform or arguments used at environment creation time). +It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory +(on Windows, this is :file:`Lib\site-packages`). +If an existing directory is specified, it will be re-used. + +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + +.. deprecated-removed:: 3.6 3.8 + :program:`pyvenv` was the recommended tool for creating virtual environments + for Python 3.3 and 3.4, and replaced in 3.5 by executing ``venv`` directly. + +.. highlight:: none + +On Windows, invoke the ``venv`` command as follows: + +.. code-block:: ps1con + + PS> python -m venv C:\path\to\new\virtual\environment + +The command, if run with ``-h``, will show the available options:: + + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-files] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory + if it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been + upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-files + Skips adding SCM ignore files to the environment + directory (Git is supported by default). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options. + +.. versionchanged:: 3.4 + In earlier versions, if the target directory already existed, an error was + raised, unless the ``--clear`` or ``--upgrade`` option was provided. + +.. versionchanged:: 3.9 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI. + +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + +.. versionchanged:: 3.13 + + Added the ``--without-scm-ignore-files`` option. +.. versionchanged:: 3.13 + ``venv`` now creates a :file:`.gitignore` file for Git by default. + +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + +.. note:: + On Microsoft Windows, it may be required to enable the ``Activate.ps1`` + script by setting the execution policy for the user. You can do this by + issuing the following PowerShell command: + + .. code-block:: powershell + + PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + + See `About Execution Policies + `_ + for more information. + +The created :file:`pyvenv.cfg` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + +Multiple paths can be given to ``venv``, in which case an identical virtual +environment will be created, according to the given options, at each provided +path. .. _venv-explanation: @@ -95,7 +215,7 @@ containing the virtual environment): | +------------+--------------------------------------------------+ | | csh/tcsh | :samp:`$ source {}/bin/activate.csh` | | +------------+--------------------------------------------------+ -| | PowerShell | :samp:`$ {}/bin/Activate.ps1` | +| | pwsh | :samp:`$ {}/bin/Activate.ps1` | +-------------+------------+--------------------------------------------------+ | Windows | cmd.exe | :samp:`C:\\> {}\\Scripts\\activate.bat` | | +------------+--------------------------------------------------+ @@ -117,7 +237,7 @@ should be runnable without activating it. In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{}/bin/python`. +:samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed @@ -168,31 +288,31 @@ creation according to their needs, the :class:`EnvBuilder` class. The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: - * ``system_site_packages`` -- a Boolean value indicating that the system Python + * *system_site_packages* -- a boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete the contents of + * *clear* -- a boolean value which, if true, will delete the contents of any existing target directory, before creating the environment. - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + * *symlinks* -- a boolean value indicating whether to attempt to symlink the Python binary rather than copying. - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + * *upgrade* -- a boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). - * ``with_pip`` -- a Boolean value which, if true, ensures pip is + * *with_pip* -- a boolean value which, if true, ensures pip is installed in the virtual environment. This uses :mod:`ensurepip` with the ``--default-pip`` option. - * ``prompt`` -- a String to be used after virtual environment is activated + * *prompt* -- a string to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would be used). If the special string ``"."`` is provided, the basename of the current directory is used as the prompt. - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * *upgrade_deps* -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_files`` -- Create ignore files based for the specified source + * *scm_ignore_files* -- Create ignore files based for the specified source control managers (SCM) in the iterable. Support is defined by having a method named ``create_{scm}_ignore_file``. The only value supported by default is ``"git"`` via :meth:`create_git_ignore_file`. @@ -210,10 +330,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: + :class:`EnvBuilder` may be used as a base class. .. method:: create(env_dir) @@ -313,14 +430,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip``) + Upgrades the core venv dependency packages (currently :pypi:`pip`) in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 .. versionchanged:: 3.12 - ``setuptools`` is no longer a core venv dependency. + :pypi:`setuptools` is no longer a core venv dependency. .. method:: post_setup(context) @@ -328,25 +445,15 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - .. method:: install_scripts(context, path) + This method can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the + ``common``, ``posix``, ``nt``; each containing scripts destined for the + ``bin`` directory in the environment. The contents of ``common`` and the directory corresponding to :data:`os.name` are copied after some text replacement of placeholders: @@ -371,10 +478,20 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: create_git_ignore_file(context) Creates a ``.gitignore`` file within the virtual environment that causes - the entire directory to be ignored by the ``git`` source control manager. + the entire directory to be ignored by the Git source control manager. .. versionadded:: 3.13 + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ @@ -387,16 +504,16 @@ There is also a module-level convenience function: .. versionadded:: 3.3 .. versionchanged:: 3.4 - Added the ``with_pip`` parameter + Added the *with_pip* parameter .. versionchanged:: 3.6 - Added the ``prompt`` parameter + Added the *prompt* parameter .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter + Added the *upgrade_deps* parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_files`` parameter + Added the *scm_ignore_files* parameter An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 68b9ff5ce2f78c..5ea65cbd8ca94c 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -597,6 +597,9 @@ Available Context Managers passed to :func:`simplefilter` as if it were called immediately on entering the context. + See :ref:`warning-filter` for the meaning of the *category* and *lineno* + parameters. + .. note:: The :class:`catch_warnings` manager works by replacing and diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index d31fbf87b739dc..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1217,9 +1217,10 @@ A function definition defines a user-defined function object (see section : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + parameter_list_starargs: "*" [`star_parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] + star_parameter: `identifier` [":" ["*"] `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -1326,11 +1327,16 @@ and may only be passed by positional arguments. Parameters may have an :term:`annotation ` of the form "``: expression``" following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of +``*identifier`` or ``**identifier``. (As a special case, parameters of the form +``*identifier`` may have an annotation "``: *expression``".) Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the semantics of a function. See :ref:`annotations` for more information on annotations. +.. versionchanged:: 3.11 + Parameters of the form "``*identifier``" may have an annotation + "``: *expression``". See :pep:`646`. + .. index:: pair: lambda; expression It is also possible to create anonymous functions (functions not bound to a @@ -1416,7 +1422,7 @@ dictionary. The class name is bound to this class object in the original local namespace. The order in which attributes are defined in the class body is preserved -in the new class's ``__dict__``. Note that this is reliable only right +in the new class's :attr:`~type.__dict__`. Note that this is reliable only right after the class is created and only for classes that were defined using the definition syntax. @@ -1447,8 +1453,8 @@ decorators. The result is then bound to the class name. A list of :ref:`type parameters ` may be given in square brackets immediately after the class's name. This indicates to static type checkers that the class is generic. At runtime, -the type parameters can be retrieved from the class's ``__type_params__`` -attribute. See :ref:`generic-classes` for more. +the type parameters can be retrieved from the class's +:attr:`~type.__type_params__` attribute. See :ref:`generic-classes` for more. .. versionchanged:: 3.12 Type parameter lists are new in Python 3.12. @@ -1661,8 +1667,8 @@ with more precision. The scope of type parameters is modeled with a special function (technically, an :ref:`annotation scope `) that wraps the creation of the generic object. -Generic functions, classes, and type aliases have a :attr:`!__type_params__` -attribute listing their type parameters. +Generic functions, classes, and type aliases have a +:attr:`~definition.__type_params__` attribute listing their type parameters. Type parameters come in three kinds: @@ -1924,5 +1930,5 @@ all annotations are instead stored as strings:: therefore the function's :term:`docstring`. .. [#] A string literal appearing as the first statement in the class body is - transformed into the namespace's ``__doc__`` item and therefore the class's - :term:`docstring`. + transformed into the namespace's :attr:`~type.__doc__` item and therefore + the class's :term:`docstring`. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 21aee0b6d0e3c5..f56bd5e8a7803a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -564,8 +564,9 @@ Special read-only attributes in which the function was defined. * - .. attribute:: function.__closure__ - - ``None`` or a :class:`tuple` of cells that contain bindings for the - function's free variables. + - ``None`` or a :class:`tuple` of cells that contain bindings for the names specified + in the :attr:`~codeobject.co_freevars` attribute of the function's + :attr:`code object `. A cell object has the attribute ``cell_contents``. This can be used to get the value of the cell, as well as set the value. @@ -595,7 +596,6 @@ Most of these attributes check the type of the assigned value: * - .. attribute:: function.__doc__ - The function's documentation string, or ``None`` if unavailable. - Not inherited by subclasses. * - .. attribute:: function.__name__ - The function's name. @@ -846,6 +846,7 @@ this case, the special read-only attribute :attr:`!__self__` is set to the objec denoted by *alist*. (The attribute has the same semantics as it does with :attr:`other instance methods `.) +.. _classes: Classes ^^^^^^^ @@ -864,6 +865,8 @@ Instances of arbitrary classes can be made callable by defining a :meth:`~object.__call__` method in their class. +.. _module-objects: + Modules ------- @@ -889,58 +892,246 @@ Attribute assignment updates the module's namespace dictionary, e.g., .. index:: single: __name__ (module attribute) - single: __doc__ (module attribute) + single: __spec__ (module attribute) + single: __package__ (module attribute) + single: __loader__ (module attribute) + single: __path__ (module attribute) single: __file__ (module attribute) + single: __cached__ (module attribute) + single: __doc__ (module attribute) single: __annotations__ (module attribute) single: __annotate__ (module attribute) pair: module; namespace -Predefined (writable) attributes: +.. _import-mod-attrs: + +Import-related attributes on module objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Module objects have the following attributes that relate to the +:ref:`import system `. When a module is created using the machinery associated +with the import system, these attributes are filled in based on the module's +:term:`spec `, before the :term:`loader` executes and loads the +module. + +To create a module dynamically rather than using the import system, +it's recommended to use :func:`importlib.util.module_from_spec`, +which will set the various import-controlled attributes to appropriate values. +It's also possible to use the :class:`types.ModuleType` constructor to create +modules directly, but this technique is more error-prone, as most attributes +must be manually set on the module object after it has been created when using +this approach. + +.. caution:: + + With the exception of :attr:`~module.__name__`, it is **strongly** + recommended that you rely on :attr:`~module.__spec__` and its attributes + instead of any of the other individual attributes listed in this subsection. + Note that updating an attribute on :attr:`!__spec__` will not update the + corresponding attribute on the module itself: + + .. doctest:: + + >>> import typing + >>> typing.__name__, typing.__spec__.name + ('typing', 'typing') + >>> typing.__spec__.name = 'spelling' + >>> typing.__name__, typing.__spec__.name + ('typing', 'spelling') + >>> typing.__name__ = 'keyboard_smashing' + >>> typing.__name__, typing.__spec__.name + ('keyboard_smashing', 'spelling') + +.. attribute:: module.__name__ + + The name used to uniquely identify the module in the import system. + For a directly executed module, this will be set to ``"__main__"``. + + This attribute must be set to the fully qualified name of the module. + It is expected to match the value of + :attr:`module.__spec__.name `. + +.. attribute:: module.__spec__ + + A record of the module's import-system-related state. + + Set to the :class:`module spec ` that was + used when importing the module. See :ref:`module-specs` for more details. + + .. versionadded:: 3.4 + +.. attribute:: module.__package__ + + The :term:`package` a module belongs to. + + If the module is top-level (that is, not a part of any specific package) + then the attribute should be set to ``''`` (the empty string). Otherwise, + it should be set to the name of the module's package (which can be equal to + :attr:`module.__name__` if the module itself is a package). See :pep:`366` + for further details. - :attr:`__name__` - The module's name. + This attribute is used instead of :attr:`~module.__name__` to calculate + explicit relative imports for main modules. It defaults to ``None`` for + modules created dynamically using the :class:`types.ModuleType` constructor; + use :func:`importlib.util.module_from_spec` instead to ensure the attribute + is set to a :class:`str`. - :attr:`__doc__` - The module's documentation string, or ``None`` if - unavailable. + It is **strongly** recommended that you use + :attr:`module.__spec__.parent ` + instead of :attr:`!module.__package__`. :attr:`__package__` is now only used + as a fallback if :attr:`!__spec__.parent` is not set, and this fallback + path is deprecated. - :attr:`__file__` - The pathname of the file from which the - module was loaded, if it was loaded from a file. - The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter. For extension modules - loaded dynamically from a shared library, it's the pathname of the shared - library file. + .. versionchanged:: 3.4 + This attribute now defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor. + Previously the attribute was optional. + + .. versionchanged:: 3.6 + The value of :attr:`!__package__` is expected to be the same as + :attr:`__spec__.parent `. + :attr:`__package__` is now only used as a fallback during import + resolution if :attr:`!__spec__.parent` is not defined. + + .. versionchanged:: 3.10 + :exc:`ImportWarning` is raised if an import resolution falls back to + :attr:`!__package__` instead of + :attr:`__spec__.parent `. + + .. versionchanged:: 3.12 + Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` when + falling back to :attr:`!__package__` during import resolution. + + .. deprecated-removed:: 3.13 3.15 + :attr:`!__package__` will cease to be set or taken into consideration + by the import system or standard library. + +.. attribute:: module.__loader__ + + The :term:`loader` object that the import machinery used to load the module. + + This attribute is mostly useful for introspection, but can be used for + additional loader-specific functionality, for example getting data + associated with a loader. + + :attr:`!__loader__` defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor; + use :func:`importlib.util.module_from_spec` instead to ensure the attribute + is set to a :term:`loader` object. + + It is **strongly** recommended that you use + :attr:`module.__spec__.loader ` + instead of :attr:`!module.__loader__`. + + .. versionchanged:: 3.4 + This attribute now defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor. + Previously the attribute was optional. + + .. deprecated-removed:: 3.12 3.14 + Setting :attr:`!__loader__` on a module while failing to set + :attr:`!__spec__.loader` is deprecated. In Python 3.14, + :attr:`!__loader__` will cease to be set or taken into consideration by + the import system or the standard library. + +.. attribute:: module.__path__ + + A (possibly empty) :term:`sequence` of strings enumerating the locations + where the package's submodules will be found. Non-package modules should + not have a :attr:`!__path__` attribute. See :ref:`package-path-rules` for + more details. + + It is **strongly** recommended that you use + :attr:`module.__spec__.submodule_search_locations ` + instead of :attr:`!module.__path__`. + +.. attribute:: module.__file__ +.. attribute:: module.__cached__ + + :attr:`!__file__` and :attr:`!__cached__` are both optional attributes that + may or may not be set. Both attributes should be a :class:`str` when they + are available. + + :attr:`!__file__` indicates the pathname of the file from which the module + was loaded (if loaded from a file), or the pathname of the shared library + file for extension modules loaded dynamically from a shared library. + It might be missing for certain types of modules, such as C modules that are + statically linked into the interpreter, and the + :ref:`import system ` may opt to leave it unset if it + has no semantic meaning (for example, a module loaded from a database). + + If :attr:`!__file__` is set then the :attr:`!__cached__` attribute might + also be set, which is the path to any compiled version of + the code (for example, a byte-compiled file). The file does not need to exist + to set this attribute; the path can simply point to where the + compiled file *would* exist (see :pep:`3147`). + + Note that :attr:`!__cached__` may be set even if :attr:`!__file__` is not + set. However, that scenario is quite atypical. Ultimately, the + :term:`loader` is what makes use of the module spec provided by the + :term:`finder` (from which :attr:`!__file__` and :attr:`!__cached__` are + derived). So if a loader can load from a cached module but otherwise does + not load from a file, that atypical scenario may be appropriate. + + It is **strongly** recommended that you use + :attr:`module.__spec__.cached ` + instead of :attr:`!module.__cached__`. + + .. deprecated-removed:: 3.13 3.15 + Setting :attr:`!__cached__` on a module while failing to set + :attr:`!__spec__.cached` is deprecated. In Python 3.15, + :attr:`!__cached__` will cease to be set or taken into consideration by + the import system or standard library. + +Other writable attributes on module objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As well as the import-related attributes listed above, module objects also have +the following writable attributes: + +.. attribute:: module.__doc__ - :attr:`~object.__annotations__` - A dictionary containing - :term:`variable annotations ` collected during - module body execution. For best practices on working - with :attr:`!__annotations__`, see :mod:`annotationlib`. + The module's documentation string, or ``None`` if unavailable. + See also: :attr:`__doc__ attributes `. - .. versionchanged:: 3.14 - Annotations are now :ref:`lazily evaluated `. - See :pep:`649`. +.. attribute:: module.__annotations__ + + A dictionary containing :term:`variable annotations ` + collected during module body execution. For best practices on working with + :attr:`!__annotations__`, see :mod:`annotationlib`. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. - :attr:`~object.__annotate__` - The :term:`annotate function` for this module, or ``None`` - if the module has no annotations. See :attr:`object.__annotate__`. +.. attribute:: module.__annotate__ - .. versionadded:: 3.14 + The :term:`annotate function` for this module, or ``None`` if the module has + no annotations. See also: :attr:`~object.__annotate__` attributes. + + .. versionadded:: 3.14 + +Module dictionaries +^^^^^^^^^^^^^^^^^^^ + +Module objects also have the following special read-only attribute: .. index:: single: __dict__ (module attribute) +.. attribute:: module.__dict__ -Special read-only attribute: :attr:`~object.__dict__` is the module's -namespace as a dictionary object. + The module's namespace as a dictionary object. Uniquely among the attributes + listed here, :attr:`!__dict__` cannot be accessed as a global variable from + within a module; it can only be accessed as an attribute on module objects. -.. impl-detail:: + .. impl-detail:: + + Because of the way CPython clears module dictionaries, the module + dictionary will be cleared when the module falls out of scope even if the + dictionary still has live references. To avoid this, copy the dictionary + or keep the module around while using its dictionary directly. - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. +.. _class-attrs-and-methods: Custom classes -------------- @@ -984,6 +1175,9 @@ of a base class. A class object can be called (see above) to yield a class instance (see below). +Special attributes +^^^^^^^^^^^^^^^^^^ + .. index:: single: __name__ (class attribute) single: __module__ (class attribute) @@ -996,66 +1190,121 @@ A class object can be called (see above) to yield a class instance (see below). single: __static_attributes__ (class attribute) single: __firstlineno__ (class attribute) -Special attributes: +.. list-table:: + :header-rows: 1 - :attr:`~definition.__name__` - The class name. + * - Attribute + - Meaning - :attr:`__module__` - The name of the module in which the class was defined. + * - .. attribute:: type.__name__ + - The class's name. + See also: :attr:`__name__ attributes `. - :attr:`~object.__dict__` - The dictionary containing the class's namespace. + * - .. attribute:: type.__qualname__ + - The class's :term:`qualified name`. + See also: :attr:`__qualname__ attributes `. - :attr:`~class.__bases__` - A tuple containing the base classes, in the order of - their occurrence in the base class list. + * - .. attribute:: type.__module__ + - The name of the module in which the class was defined. - :attr:`__doc__` - The class's documentation string, or ``None`` if undefined. + * - .. attribute:: type.__dict__ + - A :class:`mapping proxy ` + providing a read-only view of the class's namespace. + See also: :attr:`__dict__ attributes `. - :attr:`~object.__annotations__` - A dictionary containing - :term:`variable annotations ` - collected during class body execution. For best practices on - working with :attr:`~object.__annotations__`, please see - :mod:`annotationlib`. + * - .. attribute:: type.__bases__ + - A :class:`tuple` containing the class's bases. + In most cases, for a class defined as ``class X(A, B, C)``, + ``X.__bases__`` will be exactly equal to ``(A, B, C)``. - .. warning:: + * - .. attribute:: type.__doc__ + - The class's documentation string, or ``None`` if undefined. + Not inherited by subclasses. - Accessing the :attr:`~object.__annotations__` attribute of a class - object directly may yield incorrect results in the presence of - metaclasses. Use :func:`annotationlib.get_annotations` to - retrieve class annotations safely. + * - .. attribute:: type.__annotations__ + - A dictionary containing + :term:`variable annotations ` + collected during class body execution. See also: + :attr:`__annotations__ attributes `. - .. versionchanged:: 3.14 - Annotations are now :ref:`lazily evaluated `. - See :pep:`649`. + For best practices on working with :attr:`~object.__annotations__`, + please see :mod:`annotationlib`. - :attr:`~object.__annotate__` - The :term:`annotate function` for this class, or ``None`` - if the class has no annotations. See :attr:`object.__annotate__`. + .. caution:: - .. warning:: + Accessing the :attr:`!__annotations__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. In addition, the attribute may not exist for + some classes. Use :func:`annotationlib.get_annotations` to + retrieve class annotations safely. - Accessing the :attr:`~object.__annotate__` attribute of a class - object directly may yield incorrect results in the presence of - metaclasses. Use :func:`annotationlib.get_annotate_function` to - retrieve the annotate function safely. + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. - .. versionadded:: 3.14 + * - .. method:: type.__annotate__ + - The :term:`annotate function` for this class, or ``None`` + if the class has no annotations. + See also: :attr:`__annotate__ attributes `. - :attr:`__type_params__` - A tuple containing the :ref:`type parameters ` of - a :ref:`generic class `. + .. caution:: - :attr:`~class.__static_attributes__` - A tuple containing names of attributes of this class which are assigned - through ``self.X`` from any function in its body. + Accessing the :attr:`!__annotate__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. Use :func:`annotationlib.get_annotate_function` to + retrieve the annotate function safely. - :attr:`__firstlineno__` - The line number of the first line of the class definition, including decorators. + .. versionadded:: 3.14 + * - .. attribute:: type.__type_params__ + - A :class:`tuple` containing the :ref:`type parameters ` of + a :ref:`generic class `. + + .. versionadded:: 3.12 + + * - .. attribute:: type.__static_attributes__ + - A :class:`tuple` containing names of attributes of this class which are + assigned through ``self.X`` from any function in its body. + + .. versionadded:: 3.13 + + * - .. attribute:: type.__firstlineno__ + - The line number of the first line of the class definition, + including decorators. + Setting the :attr:`__module__` attribute removes the + :attr:`!__firstlineno__` item from the type's dictionary. + + .. versionadded:: 3.13 + + * - .. attribute:: type.__mro__ + - The :class:`tuple` of classes that are considered when looking for + base classes during method resolution. + + +Special methods +^^^^^^^^^^^^^^^ + +In addition to the special attributes described above, all Python classes also +have the following two methods available: + +.. method:: type.mro + + This method can be overridden by a metaclass to customize the method + resolution order for its instances. It is called at class instantiation, + and its result is stored in :attr:`~type.__mro__`. + +.. method:: type.__subclasses__ + + Each class keeps a list of weak references to its immediate subclasses. This + method returns a list of all those references still alive. The list is in + definition order. Example: + + .. doctest:: + + >>> class A: pass + >>> class B(A): pass + >>> A.__subclasses__() + [] Class instances --------------- @@ -1095,12 +1344,22 @@ dictionary directly. Class instances can pretend to be numbers, sequences, or mappings if they have methods with certain special names. See section :ref:`specialnames`. +Special attributes +^^^^^^^^^^^^^^^^^^ + .. index:: single: __dict__ (instance attribute) single: __class__ (instance attribute) -Special attributes: :attr:`~object.__dict__` is the attribute dictionary; -:attr:`~instance.__class__` is the instance's class. +.. attribute:: object.__class__ + + The class to which a class instance belongs. + +.. attribute:: object.__dict__ + + A dictionary or other mapping object used to store an object's (writable) + attributes. Not all instances have a :attr:`!__dict__` attribute; see the + section on :ref:`slots` for more details. I/O objects (also known as file objects) @@ -1215,10 +1474,14 @@ Special read-only attributes * - .. attribute:: codeobject.co_cellvars - A :class:`tuple` containing the names of :ref:`local variables ` - that are referenced by nested functions inside the function + that are referenced from at least one :term:`nested scope` inside the function * - .. attribute:: codeobject.co_freevars - - A :class:`tuple` containing the names of free variables in the function + - A :class:`tuple` containing the names of + :term:`free (closure) variables ` that a :term:`nested scope` + references in an outer scope. See also :attr:`function.__closure__`. + + Note: references to global and builtin names are *not* included. * - .. attribute:: codeobject.co_code - A string representing the sequence of :term:`bytecode` instructions in @@ -2330,9 +2593,9 @@ Notes on using *__slots__*: * The action of a *__slots__* declaration is not limited to the class where it is defined. *__slots__* declared in parents are available in - child classes. However, child subclasses will get a :attr:`~object.__dict__` and - *__weakref__* unless they also define *__slots__* (which should only - contain names of any *additional* slots). + child classes. However, instances of a child subclass will get a + :attr:`~object.__dict__` and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* slots). * If a class defines a slot also defined in a base class, the instance variable defined by the base class slot is inaccessible (except by retrieving its @@ -2351,7 +2614,7 @@ Notes on using *__slots__*: to provide per-attribute docstrings that will be recognised by :func:`inspect.getdoc` and displayed in the output of :func:`help`. -* :attr:`~instance.__class__` assignment works only if both classes have the +* :attr:`~object.__class__` assignment works only if both classes have the same *__slots__*. * :ref:`Multiple inheritance ` with multiple slotted parent @@ -2617,7 +2880,7 @@ in the local namespace as the defined class. When a new class is created by ``type.__new__``, the object provided as the namespace parameter is copied to a new ordered mapping and the original object is discarded. The new copy is wrapped in a read-only proxy, which -becomes the :attr:`~object.__dict__` attribute of the class object. +becomes the :attr:`~type.__dict__` attribute of the class object. .. seealso:: @@ -2645,14 +2908,14 @@ order to allow the addition of Abstract Base Classes (ABCs) as "virtual base classes" to any class or type (including built-in types), including other ABCs. -.. method:: class.__instancecheck__(self, instance) +.. method:: type.__instancecheck__(self, instance) Return true if *instance* should be considered a (direct or indirect) instance of *class*. If defined, called to implement ``isinstance(instance, class)``. -.. method:: class.__subclasscheck__(self, subclass) +.. method:: type.__subclasscheck__(self, subclass) Return true if *subclass* should be considered a (direct or indirect) subclass of *class*. If defined, called to implement ``issubclass(subclass, @@ -2668,8 +2931,8 @@ case the instance is itself a class. :pep:`3119` - Introducing Abstract Base Classes Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and - :meth:`~class.__subclasscheck__`, with motivation for this functionality + :func:`issubclass` behavior through :meth:`~type.__instancecheck__` and + :meth:`~type.__subclasscheck__`, with motivation for this functionality in the context of adding Abstract Base Classes (see the :mod:`abc` module) to the language. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index a02b5153ef0620..cb6c524dd97a30 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -90,7 +90,7 @@ If a name is bound in a block, it is a local variable of that block, unless declared as :keyword:`nonlocal` or :keyword:`global`. If a name is bound at the module level, it is a global variable. (The variables of the module code block are local and global.) If a variable is used in a code block but not -defined there, it is a :dfn:`free variable`. +defined there, it is a :term:`free variable`. Each occurrence of a name in the program text refers to the :dfn:`binding` of that name established by the following name resolution rules. @@ -226,8 +226,8 @@ Annotation scopes differ from function scopes in the following ways: statements in inner scopes. This includes only type parameters, as no other syntactic elements that can appear within annotation scopes can introduce new names. * While annotation scopes have an internal name, that name is not reflected in the - :term:`__qualname__ ` of objects defined within the scope. - Instead, the :attr:`!__qualname__` + :term:`qualified name` of objects defined within the scope. + Instead, the :attr:`~definition.__qualname__` of such objects is as if the object were defined in the enclosing scope. .. versionadded:: 3.12 @@ -337,6 +337,9 @@ enclosing namespace, but in the global namespace. [#]_ The :func:`exec` and :func:`eval` functions have optional arguments to override the global and local namespace. If only one namespace is specified, it is used for both. +.. XXX(ncoghlan) above is only accurate for string execution. When executing code objects, + closure cells may now be passed explicitly to resolve co_freevars references. + Docs issue: https://github.com/python/cpython/issues/122826 .. _exceptions: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1ed715109ca5f7..decde0d297cf59 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -104,8 +104,8 @@ identifier is used but only the following private identifiers are mangled: - Any name used as the name of a variable that is assigned or read or any name of an attribute being accessed. - The ``__name__`` attribute of nested functions, classes, and type aliases - is however not mangled. + The :attr:`~definition.__name__` attribute of nested functions, classes, and + type aliases is however not mangled. - The name of imported modules, e.g., ``__spam`` in ``import __spam``. If the module is part of a package (i.e., its name contains a dot), @@ -284,7 +284,7 @@ A list display is a possibly empty series of expressions enclosed in square brackets: .. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" + list_display: "[" [`flexible_expression_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either a list of expressions or a comprehension. When a comma-separated list of @@ -309,7 +309,7 @@ A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: .. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" + set_display: "{" (`flexible_expression_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by either a sequence of expressions or a comprehension. When a comma-separated @@ -454,7 +454,7 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_from: "yield" "from" `expression` - yield_expression: "yield" `expression_list` | `yield_from` + yield_expression: "yield" `yield_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -485,9 +485,9 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` +suspended again, returning the value of :token:`~python-grammar:yield_list` to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. +or ``None`` if :token:`~python-grammar:yield_list` is omitted. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -576,7 +576,7 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + :token:`~python-grammar:yield_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -695,7 +695,7 @@ how a generator object would be used in a :keyword:`for` statement. Calling one of the asynchronous generator's methods returns an :term:`awaitable` object, and the execution starts when this object is awaited on. At that time, the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the +again, returning the value of :token:`~python-grammar:yield_list` to the awaiting coroutine. As with a generator, suspension means that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -759,7 +759,7 @@ which are used to control the execution of a generator function. asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the + expression. The value of the :token:`~python-grammar:yield_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a @@ -892,7 +892,7 @@ will generally select an element from the container. The subscription of a :ref:`GenericAlias ` object. .. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" + subscription: `primary` "[" `flexible_expression_list` "]" When an object is subscripted, the interpreter will evaluate the primary and the expression list. @@ -904,9 +904,13 @@ primary is subscripted, the evaluated result of the expression list will be passed to one of these methods. For more details on when ``__class_getitem__`` is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. +If the expression list contains at least one comma, or if any of the expressions +are starred, the expression list will evaluate to a :class:`tuple` containing +the items of the expression list. Otherwise, the expression list will evaluate +to the value of the list's sole member. + +.. versionchanged:: 3.11 + Expressions in an expression list may be starred. See :pep:`646`. For built-in objects, there are two types of objects that support subscription via :meth:`~object.__getitem__`: @@ -1803,6 +1807,9 @@ returns a boolean value regardless of the type of its argument single: assignment expression single: walrus operator single: named expression + pair: assignment; expression + +.. _assignment-expressions: Assignment expressions ====================== @@ -1905,10 +1912,12 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar + starred_expression: ["*"] `or_expr` + flexible_expression: `assignment_expression` | `starred_expression` + flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] + starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` + yield_list: `expression_list` | `starred_expression` "," [`starred_expression_list`] .. index:: pair: object; tuple @@ -1929,6 +1938,9 @@ the unpacking. .. versionadded:: 3.5 Iterable unpacking in expression lists, originally proposed by :pep:`448`. +.. versionadded:: 3.11 + Any item in an expression list may be starred. See :pep:`646`. + .. index:: pair: trailing; comma A trailing comma is required only to create a one-item tuple, diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 19b8aa05072c73..ac363e8cfa00dc 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -513,8 +513,10 @@ holding is that if you have ``sys.modules['spam']`` and ``sys.modules['spam.foo']`` (as you would after the above import), the latter must appear as the ``foo`` attribute of the former. -Module spec ------------ +.. _module-specs: + +Module specs +------------ The import machinery uses a variety of information about each module during import, especially before loading. Most of the information is @@ -527,163 +529,44 @@ and the loader that executes it. Most importantly, it allows the import machinery to perform the boilerplate operations of loading, whereas without a module spec the loader had that responsibility. -The module's spec is exposed as the ``__spec__`` attribute on a module object. +The module's spec is exposed as :attr:`module.__spec__`. Setting +:attr:`!__spec__` appropriately applies equally to +:ref:`modules initialized during interpreter startup `. +The one exception is ``__main__``, where :attr:`!__spec__` is +:ref:`set to None in some cases `. + See :class:`~importlib.machinery.ModuleSpec` for details on the contents of the module spec. .. versionadded:: 3.4 -.. _import-mod-attrs: - -Import-related module attributes --------------------------------- - -The import machinery fills in these attributes on each module object -during loading, based on the module's spec, before the loader executes -the module. - -It is **strongly** recommended that you rely on :attr:`__spec__` and -its attributes instead of any of the other individual attributes -listed below. - -.. attribute:: __name__ - - The ``__name__`` attribute must be set to the fully qualified name of - the module. This name is used to uniquely identify the module in - the import system. - -.. attribute:: __loader__ - - The ``__loader__`` attribute must be set to the loader object that - the import machinery used when loading the module. This is mostly - for introspection, but can be used for additional loader-specific - functionality, for example getting data associated with a loader. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of this attribute. - - .. versionchanged:: 3.12 - The value of ``__loader__`` is expected to be the same as - ``__spec__.loader``. The use of ``__loader__`` is deprecated and slated - for removal in Python 3.14. - -.. attribute:: __package__ - - The module's ``__package__`` attribute may be set. Its value must - be a string, but it can be the same value as its ``__name__``. When - the module is a package, its ``__package__`` value should be set to - its ``__name__``. When the module is not a package, ``__package__`` - should be set to the empty string for top-level modules, or for - submodules, to the parent package's name. See :pep:`366` for further - details. - - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of this attribute. - - .. versionchanged:: 3.6 - The value of ``__package__`` is expected to be the same as - ``__spec__.parent``. - - .. versionchanged:: 3.10 - :exc:`ImportWarning` is raised if import falls back to - ``__package__`` instead of - :attr:`~importlib.machinery.ModuleSpec.parent`. - - .. versionchanged:: 3.12 - Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` - when falling back to ``__package__``. - - .. deprecated-removed:: 3.13 3.15 - ``__package__`` will cease to be set or taken into consideration - by the import system or standard library. - - -.. attribute:: __spec__ - - The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. Setting ``__spec__`` - appropriately applies equally to :ref:`modules initialized during - interpreter startup `. The one exception is ``__main__``, - where ``__spec__`` is :ref:`set to None in some cases `. - - When ``__spec__.parent`` is not set, ``__package__`` is used as - a fallback. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - ``__spec__.parent`` is used as a fallback when ``__package__`` is - not defined. - -.. attribute:: __path__ - - If the module is a package (either regular or namespace), the module - object's ``__path__`` attribute must be set. The value must be - iterable, but may be empty if ``__path__`` has no further significance. - If ``__path__`` is not empty, it must produce strings when iterated - over. More details on the semantics of ``__path__`` are given - :ref:`below `. - - Non-package modules should not have a ``__path__`` attribute. - -.. attribute:: __file__ -.. attribute:: __cached__ - - ``__file__`` is optional (if set, value must be a string). It indicates - the pathname of the file from which the module was loaded (if - loaded from a file), or the pathname of the shared library file - for extension modules loaded dynamically from a shared library. - It might be missing for certain types of modules, such as C - modules that are statically linked into the interpreter, and the - import system may opt to leave it unset if it has no semantic - meaning (e.g. a module loaded from a database). - - If ``__file__`` is set then the ``__cached__`` attribute might also - be set, which is the path to any compiled version of - the code (e.g. byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file would exist (see :pep:`3147`). - - Note that ``__cached__`` may be set even if ``__file__`` is not - set. However, that scenario is quite atypical. Ultimately, the - loader is what makes use of the module spec provided by the finder - (from which ``__file__`` and ``__cached__`` are derived). So - if a loader can load from a cached module but otherwise does not load - from a file, that atypical scenario may be appropriate. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of ``__cached__``. - - .. deprecated-removed:: 3.13 3.15 - ``__cached__`` will cease to be set or taken into consideration - by the import system or standard library. - .. _package-path-rules: -module.__path__ ---------------- +__path__ attributes on modules +------------------------------ -By definition, if a module has a ``__path__`` attribute, it is a package. +The :attr:`~module.__path__` attribute should be a (possibly empty) +:term:`sequence` of strings enumerating the locations where the package's +submodules will be found. By definition, if a module has a :attr:`!__path__` +attribute, it is a :term:`package`. -A package's ``__path__`` attribute is used during imports of its subpackages. +A package's :attr:`~module.__path__` attribute is used during imports of its +subpackages. Within the import machinery, it functions much the same as :data:`sys.path`, i.e. providing a list of locations to search for modules during import. -However, ``__path__`` is typically much more constrained than -:data:`sys.path`. +However, :attr:`!__path__` is typically much more constrained than +:data:`!sys.path`. -``__path__`` must be an iterable of strings, but it may be empty. The same rules used for :data:`sys.path` also apply to a package's -``__path__``, and :data:`sys.path_hooks` (described below) are -consulted when traversing a package's ``__path__``. +:attr:`!__path__`. :data:`sys.path_hooks` (described below) are +consulted when traversing a package's :attr:`!__path__`. -A package's ``__init__.py`` file may set or alter the package's ``__path__`` +A package's ``__init__.py`` file may set or alter the package's +:attr:`~module.__path__` attribute, and this was typically the way namespace packages were implemented prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no -longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the import machinery automatically sets ``__path__`` +longer need to supply ``__init__.py`` files containing only :attr:`!__path__` +manipulation code; the import machinery automatically sets :attr:`!__path__` correctly for the namespace package. Module reprs diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ae5408ee386bbd..f7167032ad7df9 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -284,11 +284,10 @@ UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for further details. Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -are the same as in Python 2.x: the uppercase and lowercase letters ``A`` through +include the uppercase and lowercase letters ``A`` through ``Z``, the underscore ``_`` and, except for the first character, the digits ``0`` through ``9``. - -Python 3.0 introduces additional characters from outside the ASCII range (see +Python 3.0 introduced additional characters from outside the ASCII range (see :pep:`3131`). For these characters, the classification uses the version of the Unicode Character Database as included in the :mod:`unicodedata` module. diff --git a/Doc/requirements.txt b/Doc/requirements.txt index bf1028020b7af7..5105786ccf283c 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -6,7 +6,7 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -sphinx~=8.0.0 +sphinx~=8.1.0 blurb diff --git a/Doc/tools/extensions/availability.py b/Doc/tools/extensions/availability.py new file mode 100644 index 00000000000000..47833fdcb87590 --- /dev/null +++ b/Doc/tools/extensions/availability.py @@ -0,0 +1,125 @@ +"""Support for documenting platform availability""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx import addnodes +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +logger = logging.getLogger("availability") + +# known platform, libc, and threading implementations +_PLATFORMS = frozenset({ + "AIX", + "Android", + "BSD", + "DragonFlyBSD", + "Emscripten", + "FreeBSD", + "GNU/kFreeBSD", + "iOS", + "Linux", + "macOS", + "NetBSD", + "OpenBSD", + "POSIX", + "Solaris", + "Unix", + "VxWorks", + "WASI", + "Windows", +}) +_LIBC = frozenset({ + "BSD libc", + "glibc", + "musl", +}) +_THREADING = frozenset({ + # POSIX platforms with pthreads + "pthreads", +}) +KNOWN_PLATFORMS = _PLATFORMS | _LIBC | _THREADING + + +class Availability(SphinxDirective): + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> list[nodes.container]: + title = "Availability" + refnode = addnodes.pending_xref( + title, + nodes.inline(title, title, classes=["xref", "std", "std-ref"]), + refdoc=self.env.docname, + refdomain="std", + refexplicit=True, + reftarget="availability", + reftype="ref", + refwarn=True, + ) + sep = nodes.Text(": ") + parsed, msgs = self.state.inline_text(self.arguments[0], self.lineno) + pnode = nodes.paragraph(title, "", refnode, sep, *parsed, *msgs) + self.set_source_info(pnode) + cnode = nodes.container("", pnode, classes=["availability"]) + self.set_source_info(cnode) + if self.content: + self.state.nested_parse(self.content, self.content_offset, cnode) + self.parse_platforms() + + return [cnode] + + def parse_platforms(self) -> dict[str, str | bool]: + """Parse platform information from arguments + + Arguments is a comma-separated string of platforms. A platform may + be prefixed with "not " to indicate that a feature is not available. + + Example:: + + .. availability:: Windows, Linux >= 4.2, not WASI + + Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not + parsed into separate tokens. + """ + platforms = {} + for arg in self.arguments[0].rstrip(".").split(","): + arg = arg.strip() + platform, _, version = arg.partition(" >= ") + if platform.startswith("not "): + version = False + platform = platform.removeprefix("not ") + elif not version: + version = True + platforms[platform] = version + + if unknown := set(platforms).difference(KNOWN_PLATFORMS): + logger.warning( + "Unknown platform%s or syntax '%s' in '.. availability:: %s', " + "see %s:KNOWN_PLATFORMS for a set of known platforms.", + "s" if len(platforms) != 1 else "", + " ".join(sorted(unknown)), + self.arguments[0], + __file__, + ) + + return platforms + + +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_directive("availability", Availability) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py index f2df6db47a2227..9ccaec3dd5ce0f 100644 --- a/Doc/tools/extensions/patchlevel.py +++ b/Doc/tools/extensions/patchlevel.py @@ -74,4 +74,8 @@ def get_version_info(): if __name__ == "__main__": - print(format_version_info(get_header_version_info())[1]) + short_ver, full_ver = format_version_info(get_header_version_info()) + if sys.argv[1:2] == ["--short"]: + print(short_ver) + else: + print(full_ver) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 791d9296a975e7..bcb8a421e32d09 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -24,7 +24,6 @@ from sphinx.domains.changeset import VersionChange, versionlabels, versionlabel_classes from sphinx.domains.python import PyFunction, PyMethod, PyModule from sphinx.locale import _ as sphinx_gettext -from sphinx.util import logging from sphinx.util.docutils import SphinxDirective from sphinx.writers.text import TextWriter, TextTranslator from sphinx.util.display import status_iterator @@ -108,80 +107,6 @@ def run(self): return [pnode] -# Support for documenting platform availability - -class Availability(SphinxDirective): - - has_content = True - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = True - - # known platform, libc, and threading implementations - known_platforms = frozenset({ - "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", - "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", - "Unix", "VxWorks", "WASI", "Windows", "macOS", "iOS", - # libc - "BSD libc", "glibc", "musl", - # POSIX platforms with pthreads - "pthreads", - }) - - def run(self): - availability_ref = ':ref:`Availability `: ' - avail_nodes, avail_msgs = self.state.inline_text( - availability_ref + self.arguments[0], - self.lineno) - pnode = nodes.paragraph(availability_ref + self.arguments[0], - '', *avail_nodes, *avail_msgs) - self.set_source_info(pnode) - cnode = nodes.container("", pnode, classes=["availability"]) - self.set_source_info(cnode) - if self.content: - self.state.nested_parse(self.content, self.content_offset, cnode) - self.parse_platforms() - - return [cnode] - - def parse_platforms(self): - """Parse platform information from arguments - - Arguments is a comma-separated string of platforms. A platform may - be prefixed with "not " to indicate that a feature is not available. - - Example:: - - .. availability:: Windows, Linux >= 4.2, not WASI - - Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not - parsed into separate tokens. - """ - platforms = {} - for arg in self.arguments[0].rstrip(".").split(","): - arg = arg.strip() - platform, _, version = arg.partition(" >= ") - if platform.startswith("not "): - version = False - platform = platform[4:] - elif not version: - version = True - platforms[platform] = version - - unknown = set(platforms).difference(self.known_platforms) - if unknown: - cls = type(self) - logger = logging.getLogger(cls.__qualname__) - logger.warning( - f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " - f"in '.. availability:: {self.arguments[0]}', see " - f"{__file__}:{cls.__qualname__}.known_platforms for a set " - "known platforms." - ) - - return platforms - - # Support for documenting decorators class PyDecoratorMixin(object): @@ -259,7 +184,22 @@ def run(self): return PyMethod.run(self) -# Support for documenting version of removal in deprecations +# Support for documenting version of changes, additions, deprecations + +def expand_version_arg(argument, release): + """Expand "next" to the current version""" + if argument == 'next': + return sphinx_gettext('{} (unreleased)').format(release) + return argument + + +class PyVersionChange(VersionChange): + def run(self): + # Replace the 'next' special token with the current development version + self.arguments[0] = expand_version_arg(self.arguments[0], + self.config.release) + return super().run() + class DeprecatedRemoved(VersionChange): required_arguments = 2 @@ -270,8 +210,12 @@ class DeprecatedRemoved(VersionChange): def run(self): # Replace the first two arguments (deprecated version and removed version) # with a single tuple of both versions. - version_deprecated = self.arguments[0] + version_deprecated = expand_version_arg(self.arguments[0], + self.config.release) version_removed = self.arguments.pop(1) + if version_removed == 'next': + raise ValueError( + 'deprecated-removed:: second argument cannot be `next`') self.arguments[0] = version_deprecated, version_removed # Set the label based on if we have reached the removal version @@ -334,8 +278,8 @@ def run(self): # Support for building "topic help" for pydoc pydoc_topic_labels = [ - 'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'await', + 'assert', 'assignment', 'assignment-expressions', 'async', 'atom-identifiers', + 'atom-literals', 'attribute-access', 'attribute-references', 'augassign', 'await', 'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', 'bltin-null-object', 'bltin-type-objects', 'booleans', 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', @@ -473,7 +417,10 @@ def setup(app): app.add_role('issue', issue_role) app.add_role('gh', gh_issue_role) app.add_directive('impl-detail', ImplementationDetail) - app.add_directive('availability', Availability) + app.add_directive('versionadded', PyVersionChange, override=True) + app.add_directive('versionchanged', PyVersionChange, override=True) + app.add_directive('versionremoved', PyVersionChange, override=True) + app.add_directive('deprecated', PyVersionChange, override=True) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index b4217908cc63c9..45ec436fee72d7 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -1,17 +1,19 @@ {% extends "layout.html" %} {% set title = _('Download') %} {% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} + {% set dl_base = pathto('archives', resource=True) %} + {% set dl_version = version %} {% else %} {# The link below returns HTTP 404 until the first related alpha release. This is expected; use daily documentation builds for CPython development. #} - {% set dlbase = 'https://www.python.org/ftp/python/doc/' + release %} + {% set dl_base = 'https://www.python.org/ftp/python/doc/' + release %} + {% set dl_version = release %} {% endif %} {% block body %} -

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

+

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} @@ -26,27 +28,27 @@

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

{% trans %}PDF{% endtrans %} - {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}HTML{% endtrans %} - {% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}Plain text{% endtrans %} - {% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}Texinfo{% endtrans %} - {% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}EPUB{% endtrans %} - {% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %} diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 675faa8c52477d..492568961d8a51 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -276,8 +276,8 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to -the class: ``"A simple example class"``. +:attr:`~type.__doc__` is also a valid attribute, returning the docstring +belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class object is a parameterless function that returns a new instance of the class. @@ -932,6 +932,6 @@ Examples:: .. [#] Except for one thing. Module objects have a secret read-only attribute called :attr:`~object.__dict__` which returns the dictionary used to implement the module's - namespace; the name :attr:`~object.__dict__` is an attribute but not a global name. + namespace; the name ``__dict__`` is an attribute but not a global name. Obviously, using this violates the abstraction of namespace implementation, and should be restricted to things like post-mortem debuggers. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 677d7ca02c3f2f..b830ce94ba4f47 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -160,21 +160,59 @@ arguments. In chapter :ref:`tut-structures`, we will discuss in more detail abo .. _tut-break: -:keyword:`!break` and :keyword:`!continue` Statements, and :keyword:`!else` Clauses on Loops -============================================================================================ +:keyword:`!break` and :keyword:`!continue` Statements +===================================================== The :keyword:`break` statement breaks out of the innermost enclosing -:keyword:`for` or :keyword:`while` loop. +:keyword:`for` or :keyword:`while` loop:: -A :keyword:`!for` or :keyword:`!while` loop can include an :keyword:`!else` clause. + >>> for n in range(2, 10): + ... for x in range(2, n): + ... if n % x == 0: + ... print(f"{n} equals {x} * {n//x}") + ... break + ... + 4 equals 2 * 2 + 6 equals 2 * 3 + 8 equals 2 * 4 + 9 equals 3 * 3 + +The :keyword:`continue` statement continues with the next +iteration of the loop:: + + >>> for num in range(2, 10): + ... if num % 2 == 0: + ... print(f"Found an even number {num}") + ... continue + ... print(f"Found an odd number {num}") + ... + Found an even number 2 + Found an odd number 3 + Found an even number 4 + Found an odd number 5 + Found an even number 6 + Found an odd number 7 + Found an even number 8 + Found an odd number 9 + +.. _tut-for-else: + +:keyword:`!else` Clauses on Loops +================================= + +In a :keyword:`!for` or :keyword:`!while` loop the :keyword:`!break` statement +may be paired with an :keyword:`!else` clause. If the loop finishes without +executing the :keyword:`!break`, the :keyword:`!else` clause executes. In a :keyword:`for` loop, the :keyword:`!else` clause is executed -after the loop reaches its final iteration. +after the loop finishes its final iteration, that is, if no break occurred. In a :keyword:`while` loop, it's executed after the loop's condition becomes false. -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. +In either kind of loop, the :keyword:`!else` clause is **not** executed if the +loop was terminated by a :keyword:`break`. Of course, other ways of ending the +loop early, such as a :keyword:`return` or a raised exception, will also skip +execution of the :keyword:`else` clause. This is exemplified in the following :keyword:`!for` loop, which searches for prime numbers:: @@ -198,32 +236,19 @@ which searches for prime numbers:: 9 equals 3 * 3 (Yes, this is the correct code. Look closely: the ``else`` clause belongs to -the :keyword:`for` loop, **not** the :keyword:`if` statement.) +the ``for`` loop, **not** the ``if`` statement.) -When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does with that of -:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs -when no exception occurs, and a loop's ``else`` clause runs when no ``break`` -occurs. For more on the :keyword:`!try` statement and exceptions, see -:ref:`tut-handling`. - -The :keyword:`continue` statement, also borrowed from C, continues with the next -iteration of the loop:: +One way to think of the else clause is to imagine it paired with the ``if`` +inside the loop. As the loop executes, it will run a sequence like +if/if/if/else. The ``if`` is inside the loop, encountered a number of times. If +the condition is ever true, a ``break`` will happen. If the condition is never +true, the ``else`` clause outside the loop will execute. - >>> for num in range(2, 10): - ... if num % 2 == 0: - ... print("Found an even number", num) - ... continue - ... print("Found an odd number", num) - ... - Found an even number 2 - Found an odd number 3 - Found an even number 4 - Found an odd number 5 - Found an even number 6 - Found an odd number 7 - Found an even number 8 - Found an odd number 9 +When used with a loop, the ``else`` clause has more in common with the ``else`` +clause of a :keyword:`try` statement than it does with that of ``if`` +statements: a ``try`` statement's ``else`` clause runs when no exception +occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on +the ``try`` statement and exceptions, see :ref:`tut-handling`. .. _tut-pass: @@ -436,8 +461,8 @@ Defining Functions We can create a function that writes the Fibonacci series to an arbitrary boundary:: - >>> def fib(n): # write Fibonacci series up to n - ... """Print a Fibonacci series up to n.""" + >>> def fib(n): # write Fibonacci series less than n + ... """Print a Fibonacci series less than n.""" ... a, b = 0, 1 ... while a < n: ... print(a, end=' ') @@ -807,7 +832,7 @@ parameters as there is a ``/`` in the function definition:: File "", line 1, in TypeError: pos_only_arg() got some positional-only arguments passed as keyword arguments: 'arg' -The third function ``kwd_only_args`` only allows keyword arguments as indicated +The third function ``kwd_only_arg`` only allows keyword arguments as indicated by a ``*`` in the function definition:: >>> kwd_only_arg(3) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 73f17adeea72de..31941bc112a135 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -19,13 +19,13 @@ objects: .. method:: list.append(x) :noindex: - Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. + Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. .. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items from the iterable. Equivalent to + Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. @@ -56,7 +56,7 @@ objects: .. method:: list.clear() :noindex: - Remove all items from the list. Equivalent to ``del a[:]``. + Remove all items from the list. Similar to ``del a[:]``. .. method:: list.index(x[, start[, end]]) @@ -93,7 +93,7 @@ objects: .. method:: list.copy() :noindex: - Return a shallow copy of the list. Equivalent to ``a[:]``. + Return a shallow copy of the list. Similar to ``a[:]``. An example that uses most of the list methods:: diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 054bac59c955d5..65e3b1938bca9c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -197,21 +197,19 @@ and workarounds. String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at -the end of the line. The following example:: - - print("""\ +the end of the line. In the following example, the initial newline is not +included:: + + >>> print("""\ + ... Usage: thingy [OPTIONS] + ... -h Display this usage message + ... -H hostname Hostname to connect to + ... """) Usage: thingy [OPTIONS] -h Display this usage message -H hostname Hostname to connect to - """) - -produces the following output (note that the initial newline is not included): -.. code-block:: text - - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to + >>> Strings can be concatenated (glued together) with the ``+`` operator, and repeated with ``*``:: diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 0316239e776a95..de7aa0e2342946 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -585,8 +585,9 @@ as the main module of a Python application must always use absolute imports. Packages in Multiple Directories -------------------------------- -Packages support one more special attribute, :attr:`__path__`. This is -initialized to be a list containing the name of the directory holding the +Packages support one more special attribute, :attr:`~module.__path__`. This is +initialized to be a :term:`sequence` of strings containing the name of the +directory holding the package's :file:`__init__.py` before the code in that file is executed. This variable can be modified; doing so affects future searches for modules and subpackages contained in the package. diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 91e4ce18acef1d..f362e1943b666f 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -76,7 +76,7 @@ virtual environment you're using, and modify the environment so that running ``python`` will get you that particular version and installation of Python. For example: -.. code-block:: bash +.. code-block:: console $ source ~/envs/tutorial-env/bin/activate (tutorial-env) $ python @@ -108,7 +108,7 @@ complete documentation for ``pip``.) You can install the latest version of a package by specifying a package's name: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install novas Collecting novas @@ -120,7 +120,7 @@ You can install the latest version of a package by specifying a package's name: You can also install a specific version of a package by giving the package name followed by ``==`` and the version number: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install requests==2.6.0 Collecting requests==2.6.0 @@ -133,7 +133,7 @@ version is already installed and do nothing. You can supply a different version number to get that version, or you can run ``python -m pip install --upgrade`` to upgrade the package to the latest version: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install --upgrade requests Collecting requests @@ -148,7 +148,7 @@ remove the packages from the virtual environment. ``python -m pip show`` will display information about a particular package: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip show requests --- @@ -166,7 +166,7 @@ remove the packages from the virtual environment. ``python -m pip list`` will display all of the packages installed in the virtual environment: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip list novas (3.1.1.3) @@ -179,7 +179,7 @@ the virtual environment: but the output uses the format that ``python -m pip install`` expects. A common convention is to put this list in a ``requirements.txt`` file: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip freeze > requirements.txt (tutorial-env) $ cat requirements.txt @@ -191,7 +191,7 @@ The ``requirements.txt`` can then be committed to version control and shipped as part of an application. Users can then install all the necessary packages with ``install -r``: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install -r requirements.txt Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 4976418ba33cf8..10cdf2376229ff 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -29,7 +29,7 @@ Features and minimum versions required to build CPython: * Tcl/Tk 8.5.12 for the :mod:`tkinter` module. -* Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the +* Autoconf 2.71 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. .. versionchanged:: 3.1 @@ -56,7 +56,7 @@ Features and minimum versions required to build CPython: Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. .. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. + Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 2dfac0758435d1..4b6c884f3d4f25 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -2,140 +2,223 @@ .. _using-on-mac: ********************* -Using Python on a Mac +Using Python on macOS ********************* -:Author: Bob Savage +.. sectionauthor:: Bob Savage +.. sectionauthor:: Ned Deily +This document aims to give an overview of macOS-specific behavior you should +know about to get started with Python on Mac computers. +Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, +but there are some differences in installation and some features. -Python on a Mac running macOS is in principle very similar to Python on -any other Unix platform, but there are a number of additional features such as -the integrated development environment (IDE) and the Package Manager that are -worth pointing out. +There are various ways to obtain and install Python for macOS. +Pre-built versions of the most recent versions of Python are available +from a number of distributors. Much of this document describes use of +the Pythons provided by the CPython release team for download from +the `python.org website `_. See +:ref:`alternative_bundles` for some other options. +.. |usemac_x_dot_y| replace:: 3.13 +.. |usemac_python_x_dot_y_literal| replace:: ``python3.13`` +.. |usemac_python_x_dot_y_t_literal| replace:: ``python3.13t`` +.. |usemac_python_x_dot_y_t_literal_config| replace:: ``python3.13t-config`` +.. |usemac_applications_folder_name| replace:: ``Python 3.13`` +.. |usemac_applications_folder_version| replace:: ``/Applications/Python 3.13/`` .. _getting-osx: .. _getting-and-installing-macpython: -Getting and Installing Python -============================= +Using Python for macOS from ``python.org`` +========================================== -macOS used to come with Python 2.7 pre-installed between versions -10.8 and `12.3 `_. -You are invited to install the most recent version of Python 3 from the `Python -website `__. -A current "universal2 binary" build of Python, which runs natively on the Mac's -new Apple Silicon and legacy Intel processors, is available there. +Installation steps +------------------ -What you get after installing is a number of things: +For `current Python versions `_ +(other than those in ``security`` status), the release team produces a +**Python for macOS** installer package for each new release. +A list of available installers +is available `here `_. +We recommend using the most recent supported Python version where possible. +Current installers provide a +`universal2 binary `_ build +of Python which runs natively on all Macs (Apple Silicon and Intel) that are +supported by a wide range of macOS versions, +currently typically from at least **macOS 10.13 High Sierra** on. -* A |python_version_literal| folder in your :file:`Applications` folder. In here - you find IDLE, the development environment that is a standard part of official +The downloaded file is a standard macOS installer package file (``.pkg``). +File integrity information (checksum, size, sigstore signature, etc) for each file is included +on the release download page. Installer packages and their contents are signed and notarized +with ``Python Software Foundation`` Apple Developer ID certificates +to meet `macOS Gatekeeper requirements `_. + +For a default installation, double-click on the downloaded installer package file. +This should launch the standard macOS Installer app and display the first of several +installer windows steps. + +.. image:: mac_installer_01_introduction.png + +Clicking on the **Continue** button brings up the **Read Me** for this installer. +Besides other important information, the **Read Me** documents which Python version is +going to be installed and on what versions of macOS it is supported. You may need +to scroll through to read the whole file. By default, this **Read Me** will also be +installed in |usemac_applications_folder_version| and available to read anytime. + +.. image:: mac_installer_02_readme.png + +Clicking on **Continue** proceeds to display the license for Python and for +other included software. You will then need to **Agree** to the license terms +before proceeding to the next step. This license file will also be installed +and available to be read later. + +.. image:: mac_installer_03_license.png + +After the license terms are accepted, the next step is the **Installation Type** +display. For most uses, the standard set of installation operations is appropriate. + +.. image:: mac_installer_04_installation_type.png + +By pressing the **Customize** button, you can choose to omit or select certain package +components of the installer. Click on each package name to see a description of +what it installs. +To also install support for the optional experimental free-threaded feature, +see :ref:`install-freethreaded-macos`. + +.. image:: mac_installer_05_custom_install.png + +In either case, clicking **Install** will begin the install process by asking +permission to install new software. A macOS user name with ``Administrator`` privilege +is needed as the installed Python will be available to all users of the Mac. + +When the installation is complete, the **Summary** window will appear. + +.. image:: mac_installer_06_summary.png + +Double-click on the :command:`Install Certificates.command` +icon or file in the |usemac_applications_folder_version| window to complete the +installation. + +.. image:: mac_installer_07_applications.png + +This will open a temporary :program:`Terminal` shell window that +will use the new Python to download and install SSL root certificates +for its use. + +.. image:: mac_installer_08_install_certificates.png + +If ``Successfully installed certifi`` and ``update complete`` appears +in the terminal window, the installation is complete. +Close this terminal window and the installer window. + +A default install will include: + +* A |usemac_applications_folder_name| folder in your :file:`Applications` folder. In here + you find :program:`IDLE`, the development environment that is a standard part of official Python distributions; and :program:`Python Launcher`, which handles double-clicking Python - scripts from the Finder. + scripts from the macOS `Finder `_. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell - path. To uninstall Python, you can remove these three things. A - symlink to the Python executable is placed in :file:`/usr/local/bin/`. + path. To uninstall Python, you can remove these three things. + Symlinks to the Python executable are placed in :file:`/usr/local/bin/`. .. note:: - On macOS 10.8-12.3, the Apple-provided build of Python is installed in - :file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, - respectively. You should never modify or delete these, as they are - Apple-controlled and are used by Apple- or third-party software. Remember that - if you choose to install a newer Python version from python.org, you will have - two different but functional Python installations on your computer, so it will - be important that your paths and usages are consistent with what you want to do. - -IDLE includes a Help menu that allows you to access Python documentation. If you -are completely new to Python you should start reading the tutorial introduction -in that document. - -If you are familiar with Python on other Unix platforms you should read the -section on running Python scripts from the Unix shell. - + Recent versions of macOS include a :command:`python3` command in :file:`/usr/bin/python3` + that links to a usually older and incomplete version of Python provided by and for use by + the Apple development tools, :program:`Xcode` or the :program:`Command Line Tools for Xcode`. + You should never modify or attempt to delete this installation, as it is + Apple-controlled and is used by Apple-provided or third-party software. If + you choose to install a newer Python version from ``python.org``, you will have + two different but functional Python installations on your computer that + can co-exist. The default installer options should ensure that its :command:`python3` + will be used instead of the system :command:`python3`. How to run a Python script -------------------------- -Your best way to get started with Python on macOS is through the IDLE -integrated development environment; see section :ref:`ide` and use the Help menu -when the IDE is running. +There are two ways to invoke the Python interpreter. +If you are familiar with using a Unix shell in a terminal +window, you can invoke |usemac_python_x_dot_y_literal| or ``python3`` optionally +followed by one or more command line options (described in :ref:`using-on-general`). +The Python tutorial also has a useful section on +:ref:`using Python interactively from a shell `. + +You can also invoke the interpreter through an integrated +development environment. +:ref:`idle` is a basic editor and interpreter environment +which is included with the standard distribution of Python. +:program:`IDLE` includes a Help menu that allows you to access Python documentation. If you +are completely new to Python, you can read the tutorial introduction +in that document. -If you want to run Python scripts from the Terminal window command line or from -the Finder you first need an editor to create your script. macOS comes with a -number of standard Unix command line editors, :program:`vim` -:program:`nano` among them. If you want a more Mac-like editor, -:program:`BBEdit` from Bare Bones Software (see -https://www.barebones.com/products/bbedit/index.html) are good choices, as is -:program:`TextMate` (see https://macromates.com). Other editors include -:program:`MacVim` (https://macvim.org) and :program:`Aquamacs` -(https://aquamacs.org). +There are many other editors and IDEs available, see :ref:`editors` +for more information. -To run your script from the Terminal window you must make sure that -:file:`/usr/local/bin` is in your shell search path. +To run a Python script file from the terminal window, you can +invoke the interpreter with the name of the script file: -To run your script from the Finder you have two options: + |usemac_python_x_dot_y_literal| ``myscript.py`` + +To run your script from the Finder, you can either: * Drag it to :program:`Python Launcher`. * Select :program:`Python Launcher` as the default application to open your - script (or any ``.py`` script) through the finder Info window and double-click it. + script (or any ``.py`` script) through the Finder Info window and double-click it. :program:`Python Launcher` has various preferences to control how your script is launched. Option-dragging allows you to change these for one invocation, or use - its Preferences menu to change things globally. - - -.. _osx-gui-scripts: - -Running scripts with a GUI --------------------------- - -With older versions of Python, there is one macOS quirk that you need to be -aware of: programs that talk to the Aqua window manager (in other words, -anything that has a GUI) need to be run in a special way. Use :program:`pythonw` -instead of :program:`python` to start such scripts. + its ``Preferences`` menu to change things globally. -With Python 3.9, you can use either :program:`python` or :program:`pythonw`. +Be aware that running the script directly from the macOS Finder might +produce different results than when running from a terminal window as +the script will not be run in the usual shell environment including +any setting of environment variables in shell profiles. +And, as with any other script or program, +be certain of what you are about to run. +.. _alternative_bundles: -Configuration -------------- +Alternative Distributions +========================= -Python on macOS honors all standard Unix environment variables such as -:envvar:`PYTHONPATH`, but setting these variables for programs started from the -Finder is non-standard as the Finder does not read your :file:`.profile` or -:file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's -`Technical Q&A QA1067 `__ -for details. +Besides the standard ``python.org`` for macOS installer, there are third-party +distributions for macOS that may include additional functionality. +Some popular distributions and their key features: -For more information on installation Python packages, see section -:ref:`mac-package-manager`. +`ActivePython `_ + Installer with multi-platform compatibility, documentation +`Anaconda `_ + Popular scientific modules (such as numpy, scipy, and pandas) and the + ``conda`` package manager. -.. _ide: +`Homebrew `_ + Package manager for macOS including multiple versions of Python and many + third-party Python-based packages (including numpy, scipy, and pandas). -The IDE -======= - -Python ships with the standard IDLE development environment. A good -introduction to using IDLE can be found at -https://www.hashcollision.org/hkn/python/idle_intro/index.html. +`MacPorts `_ + Another package manager for macOS including multiple versions of Python and many + third-party Python-based packages. May include pre-built versions of Python and + many packages for older versions of macOS. +Note that distributions might not include the latest versions of Python or +other libraries, and are not maintained or supported by the core Python team. .. _mac-package-manager: Installing Additional Python Packages ===================================== -This section has moved to the `Python Packaging User Guide`_. +Refer to the `Python Packaging User Guide`_ for more information. .. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ +.. _osx-gui-scripts: + .. _gui-programming-on-the-mac: GUI Programming @@ -143,36 +226,209 @@ GUI Programming There are several options for building GUI applications on the Mac with Python. -*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is -the foundation of most modern Mac development. Information on PyObjC is -available from :pypi:`pyobjc`. - The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform -Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with -macOS by Apple, and the latest version can be downloaded and installed from -https://www.activestate.com; it can also be built from source. +Tk toolkit (https://www.tcl.tk). A macOS-native version of Tk is included with +the installer. + +*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework. +Information on PyObjC is available from :pypi:`pyobjc`. -A number of alternative macOS GUI toolkits are available: +A number of alternative macOS GUI toolkits are available including: -* `PySide `__: Official Python bindings to the - `Qt GUI toolkit `__. +* `PySide `_: Official Python bindings to the + `Qt GUI toolkit `_. -* `PyQt `__: Alternative +* `PyQt `_: Alternative Python bindings to Qt. -* `Kivy `__: A cross-platform GUI toolkit that supports +* `Kivy `_: A cross-platform GUI toolkit that supports desktop and mobile platforms. -* `Toga `__: Part of the `BeeWare Project - `__; supports desktop, mobile, web and console apps. +* `Toga `_: Part of the `BeeWare Project + `_; supports desktop, mobile, web and console apps. -* `wxPython `__: A cross-platform toolkit that +* `wxPython `_: A cross-platform toolkit that supports desktop operating systems. + +Advanced Topics +=============== + +.. _install-freethreaded-macos: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +The ``python.org`` :ref:`Python for macOS ` +installer package can optionally install an additional build of +Python |usemac_x_dot_y| that supports :pep:`703`, the experimental free-threading feature +(running with the :term:`global interpreter lock` disabled). +Check the release page on ``python.org`` for possible updated information. + +Because this feature is still considered experimental, the support for it +is not installed by default. It is packaged as a separate install option, +available by clicking the **Customize** button on the **Installation Type** +step of the installer as described above. + +.. image:: mac_installer_09_custom_install_free_threaded.png + +If the box next to the **Free-threaded Python** package name is checked, +a separate :file:`PythonT.framework` will also be installed +alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. +This configuration allows a free-threaded Python |usemac_x_dot_y| build to co-exist +on your system with a traditional (GIL only) Python |usemac_x_dot_y| build with +minimal risk while installing or testing. This installation layout is itself +experimental and is subject to change in future releases. + +Known cautions and limitations: + +- The **UNIX command-line tools** package, which is selected by default, + will install links in :file:`/usr/local/bin` for |usemac_python_x_dot_y_t_literal|, + the free-threaded interpreter, and |usemac_python_x_dot_y_t_literal_config|, + a configuration utility which may be useful for package builders. + Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, + in most cases no changes to your ``PATH`` environment variables should + be needed to use |usemac_python_x_dot_y_t_literal|. + +- For this release, the **Shell profile updater** package and the + :file:`Update Shell Profile.command` in |usemac_applications_folder_version| + do not support the free-threaded package. + +- The free-threaded build and the traditional build have separate search + paths and separate :file:`site-packages` directories so, by default, + if you need a package available in both builds, it may need to be installed in both. + The free-threaded package will install a separate instance of :program:`pip` for use + with |usemac_python_x_dot_y_t_literal|. + + - To install a package using :command:`pip` without a :command:`venv`: + + |usemac_python_x_dot_y_t_literal| ``-m pip install `` + +- When working with multiple Python environments, it is usually safest and easiest + to :ref:`create and use virtual environments `. + This can avoid possible command name conflicts and confusion about which Python is in use: + + |usemac_python_x_dot_y_t_literal| ``-m venv `` + + then :command:`activate`. + +- To run a free-threaded version of IDLE: + + |usemac_python_x_dot_y_t_literal| ``-m idlelib`` + +- The interpreters in both builds respond to the same + :ref:`PYTHON environment variables ` + which may have unexpected results, for example, if you have ``PYTHONPATH`` + set in a shell profile. If necessary, there are + :ref:`command line options ` like ``-E`` + to ignore these environment variables. + +- The free-threaded build links to the third-party shared libraries, + such as ``OpenSSL`` and ``Tk``, installed in the traditional framework. + This means that both builds also share one set of trust certificates + as installed by the :command:`Install Certificates.command` script, + thus it only needs to be run once. + +- If you cannot depend on the link in ``/usr/local/bin`` pointing to the + ``python.org`` free-threaded |usemac_python_x_dot_y_t_literal| (for example, if you want + to install your own version there or some other distribution does), + you can explicitly set your shell ``PATH`` environment variable to + include the ``PythonT`` framework ``bin`` directory: + + .. code-block:: sh + + export PATH="/Library/Frameworks/PythonT.framework/Versions/3.13/bin":"$PATH" + + The traditional framework installation by default does something similar, + except for :file:`Python.framework`. Be aware that having both framework ``bin`` + directories in ``PATH`` can lead to confusion if there are duplicate names + like ``python3.13`` in both; which one is actually used depends on the order + they appear in ``PATH``. The ``which python3.x`` or ``which python3.xt`` + commands can show which path is being used. Using virtual environments + can help avoid such ambiguities. Another option might be to create + a shell :command:`alias` to the desired interpreter, like: + + .. code-block:: sh + + alias py3.13="/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13" + alias py3.13t="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t" + +Installing using the command line +--------------------------------- + +If you want to use automation to install the ``python.org`` installer package +(rather than by using the familiar macOS :program:`Installer` GUI app), +the macOS command line :command:`installer` utility lets you select non-default +options, too. If you are not familiar with :command:`installer`, it can be +somewhat cryptic (see :command:`man installer` for more information). +As an example, the following shell snippet shows one way to do it, +using the ``3.13.0b2`` release and selecting the free-threaded interpreter +option: + +.. code-block:: sh + + RELEASE="python-3.13.0b2-macos11.pkg" + + # download installer pkg + curl -O https://www.python.org/ftp/python/3.13.0/${RELEASE} + + # create installer choicechanges to customize the install: + # enable the PythonTFramework-3.13 package + # while accepting the other defaults (install all other packages) + cat > ./choicechanges.plist < + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.13 + + + + EOF + + sudo installer -pkg ./${RELEASE} -applyChoiceChangesXML ./choicechanges.plist -target / + + +You can then test that both installer builds are now available with something like: + +.. code-block:: console + + $ # test that the free-threaded interpreter was installed if the Unix Command Tools package was enabled + $ /usr/local/bin/python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # and the traditional interpreter + $ /usr/local/bin/python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # test that they are also available without the prefix if /usr/local/bin is on $PATH + $ python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + +.. note:: + + Current ``python.org`` installers only install to fixed locations like + :file:`/Library/Frameworks/`, :file:`/Applications`, and :file:`/usr/local/bin`. + You cannot use the :command:`installer` ``-domain`` option to install to + other locations. + .. _distributing-python-applications-on-the-mac: Distributing Python Applications -================================ +-------------------------------- A range of tools exist for converting your Python code into a standalone distributable application: @@ -180,12 +436,12 @@ distributable application: * :pypi:`py2app`: Supports creating macOS ``.app`` bundles from a Python project. -* `Briefcase `__: Part of the `BeeWare Project - `__; a cross-platform packaging tool that supports +* `Briefcase `_: Part of the `BeeWare Project + `_; a cross-platform packaging tool that supports creation of ``.app`` bundles on macOS, as well as managing signing and notarization. -* `PyInstaller `__: A cross-platform packaging tool that creates +* `PyInstaller `_: A cross-platform packaging tool that creates a single file or folder as a distributable artifact. App Store Compliance @@ -213,11 +469,6 @@ required if you are using the macOS App Store as a distribution channel. Other Resources =============== -The Pythonmac-SIG mailing list is an excellent support resource for Python users -and developers on the Mac: - -https://www.python.org/community/sigs/current/pythonmac-sig/ - -Another useful resource is the MacPython wiki: - -https://wiki.python.org/moin/MacPython +The `python.org Help page `_ has links to many useful resources. +The `Pythonmac-SIG mailing list `_ +is another support resource specifically for Python users and developers on the Mac. diff --git a/Doc/using/mac_installer_01_introduction.png b/Doc/using/mac_installer_01_introduction.png new file mode 100644 index 00000000000000..1999f3a3759093 Binary files /dev/null and b/Doc/using/mac_installer_01_introduction.png differ diff --git a/Doc/using/mac_installer_02_readme.png b/Doc/using/mac_installer_02_readme.png new file mode 100644 index 00000000000000..a36efaf7d50fd6 Binary files /dev/null and b/Doc/using/mac_installer_02_readme.png differ diff --git a/Doc/using/mac_installer_03_license.png b/Doc/using/mac_installer_03_license.png new file mode 100644 index 00000000000000..598c22a13d9e62 Binary files /dev/null and b/Doc/using/mac_installer_03_license.png differ diff --git a/Doc/using/mac_installer_04_installation_type.png b/Doc/using/mac_installer_04_installation_type.png new file mode 100644 index 00000000000000..9498fd06240a4e Binary files /dev/null and b/Doc/using/mac_installer_04_installation_type.png differ diff --git a/Doc/using/mac_installer_05_custom_install.png b/Doc/using/mac_installer_05_custom_install.png new file mode 100644 index 00000000000000..3a201d2f44655a Binary files /dev/null and b/Doc/using/mac_installer_05_custom_install.png differ diff --git a/Doc/using/mac_installer_06_summary.png b/Doc/using/mac_installer_06_summary.png new file mode 100644 index 00000000000000..1af6eee2c668cd Binary files /dev/null and b/Doc/using/mac_installer_06_summary.png differ diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png new file mode 100644 index 00000000000000..940219cad6f61c Binary files /dev/null and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/mac_installer_08_install_certificates.png b/Doc/using/mac_installer_08_install_certificates.png new file mode 100644 index 00000000000000..c125eeb18aa0c1 Binary files /dev/null and b/Doc/using/mac_installer_08_install_certificates.png differ diff --git a/Doc/using/mac_installer_09_custom_install_free_threaded.png b/Doc/using/mac_installer_09_custom_install_free_threaded.png new file mode 100644 index 00000000000000..0f69c55eddb228 Binary files /dev/null and b/Doc/using/mac_installer_09_custom_install_free_threaded.png differ diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc deleted file mode 100644 index 354eb1541ceac2..00000000000000 --- a/Doc/using/venv-create.inc +++ /dev/null @@ -1,121 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -command ``venv``:: - - python -m venv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation from which the command -was run (a common name for the target directory is ``.venv``). It also creates -a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy/symlink -of the Python binary/binaries (as appropriate for the platform or arguments -used at environment creation time). It also creates an (initially empty) -``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is -``Lib\site-packages``). If an existing directory is specified, it will be -re-used. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is - :ref:`deprecated in Python 3.6 `. - -.. highlight:: none - -On Windows, invoke the ``venv`` command as follows:: - - c:\>Python35\python -m venv c:\path\to\myenv - -Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for -your :ref:`Python installation `:: - - c:\>python -m venv c:\path\to\myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - [--without-scm-ignore-file] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if - it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been upgraded - in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-file - Skips adding the default SCM ignore file to the - environment directory (the default is a .gitignore - file). - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. - -.. versionchanged:: 3.13 - - ``--without-scm-ignore-file`` was added along with creating an ignore file - for ``git`` by default. - -.. versionchanged:: 3.12 - - ``setuptools`` is no longer a core venv dependency. - -.. versionchanged:: 3.9 - Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. - -.. note:: - While symlinks are supported on Windows, they are not recommended. Of - particular note is that double-clicking ``python.exe`` in File Explorer - will resolve the symlink eagerly and ignore the virtual environment. - -.. note:: - On Microsoft Windows, it may be required to enable the ``Activate.ps1`` - script by setting the execution policy for the user. You can do this by - issuing the following PowerShell command: - - PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - - See `About Execution Policies - `_ - for more information. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``venv``, in which case an identical virtual -environment will be created, according to the given options, at each provided -path. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 136236f51eb511..20d872d7639219 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -23,8 +23,9 @@ available for application-local distributions. As specified in :pep:`11`, a Python release only supports a Windows platform while Microsoft considers the platform under extended support. This means that -Python |version| supports Windows 8.1 and newer. If you require Windows 7 -support, please install Python 3.8. +Python |version| supports Windows 10 and newer. If you require Windows 7 +support, please install Python 3.8. If you require Windows 8.1 support, +please install Python 3.12. There are a number of different installers available for Windows, each with certain benefits and downsides. diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 8eafb48461a67c..f23f27c994d717 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -443,8 +443,8 @@ Python syntax:: f.grammar = "A ::= B (C D)*" The dictionary containing attributes can be accessed as the function's -:attr:`~object.__dict__`. Unlike the :attr:`~object.__dict__` attribute of class instances, in -functions you can actually assign a new dictionary to :attr:`~object.__dict__`, though +:attr:`~function.__dict__`. Unlike the :attr:`~type.__dict__` attribute of class instances, in +functions you can actually assign a new dictionary to :attr:`~function.__dict__`, though the new value is restricted to a regular Python dictionary; you *can't* be tricky and set it to a :class:`!UserDict` instance, or any other random object that behaves like a mapping. diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 5db34fa08c634a..856be5ecfa56ad 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`!__doc__` is the attribute's docstring. +* :attr:`~definition.__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,8 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`!descriptor.__get__` returns a temporary object that's +For methods, :meth:`descriptor.__get__ ` returns a temporary +object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 80849ab9a1a3db..ac463f82cfb8ca 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1113,10 +1113,10 @@ Here are all of the changes that Python 2.3 makes to the core Python language. * One of the noted incompatibilities between old- and new-style classes has been - removed: you can now assign to the :attr:`~definition.__name__` and :attr:`~class.__bases__` + removed: you can now assign to the :attr:`~type.__name__` and :attr:`~type.__bases__` attributes of new-style classes. There are some restrictions on what can be - assigned to :attr:`~class.__bases__` along the lines of those relating to assigning to - an instance's :attr:`~instance.__class__` attribute. + assigned to :attr:`!__bases__` along the lines of those relating to assigning to + an instance's :attr:`~object.__class__` attribute. .. ====================================================================== @@ -1925,8 +1925,8 @@ Changes to Python's build process and to the C API include: dependence on a system version or local installation of Expat. * If you dynamically allocate type objects in your extension, you should be - aware of a change in the rules relating to the :attr:`!__module__` and - :attr:`~definition.__name__` attributes. In summary, you will want to ensure the type's + aware of a change in the rules relating to the :attr:`~type.__module__` and + :attr:`~type.__name__` attributes. In summary, you will want to ensure the type's dictionary contains a ``'__module__'`` key; making the module name the part of the type name leading up to the final period will no longer have the desired effect. For more detail, read the API reference documentation or the source. diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 3c9c2049b89ea0..fdccfb7deb1ed7 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -502,12 +502,12 @@ Python's :option:`-m` switch allows running a module as a script. When you ran a module that was located inside a package, relative imports didn't work correctly. -The fix for Python 2.6 adds a :attr:`__package__` attribute to -modules. When this attribute is present, relative imports will be +The fix for Python 2.6 adds a :attr:`module.__package__` attribute. +When this attribute is present, relative imports will be relative to the value of this attribute instead of the -:attr:`__name__` attribute. +:attr:`~module.__name__` attribute. -PEP 302-style importers can then set :attr:`__package__` as necessary. +PEP 302-style importers can then set :attr:`~module.__package__` as necessary. The :mod:`runpy` module that implements the :option:`-m` switch now does this, so relative imports will now work correctly in scripts running from inside a package. diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 888e6279754fc2..d97f5fdd9eaa4a 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -357,8 +357,8 @@ New Syntax provides a standardized way of annotating a function's parameters and return value. There are no semantics attached to such annotations except that they can be introspected at runtime using - the :attr:`__annotations__` attribute. The intent is to encourage - experimentation through metaclasses, decorators or frameworks. + the :attr:`~object.__annotations__` attribute. The intent is to + encourage experimentation through metaclasses, decorators or frameworks. * :pep:`3102`: Keyword-only arguments. Named parameters occurring after ``*args`` in the parameter list *must* be specified using diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 9dc17494c42966..463fc269ee8fcc 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -59,7 +59,7 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.12. Brevity is key. -Python 3.12 is the latest stable release of the Python programming language, +Python 3.12 is a stable release of the Python programming language, with a mix of changes to the language and the standard library. The library changes focus on cleaning up deprecated APIs, usability, and correctness. Of note, the :mod:`!distutils` package has been removed from the standard library. @@ -1309,14 +1309,15 @@ Deprecated may be removed in a future version of Python. Use the single-arg versions of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) -* :exc:`DeprecationWarning` is now raised when ``__package__`` on a - module differs from ``__spec__.parent`` (previously it was - :exc:`ImportWarning`). +* :exc:`DeprecationWarning` is now raised when :attr:`~module.__package__` on a + module differs from + :attr:`__spec__.parent ` (previously + it was :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) -* Setting ``__package__`` or ``__cached__`` on a module is deprecated, - and will cease to be set or taken into consideration by the import system in Python 3.14. - (Contributed by Brett Cannon in :gh:`65961`.) +* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a + module is deprecated, and will cease to be set or taken into consideration by + the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.) * The bitwise inversion operator (``~``) on bool is deprecated. It will throw an error in Python 3.16. Use ``not`` for logical negation of bools instead. @@ -1887,7 +1888,7 @@ New Features The :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` flags have been added. This allows extensions classes to support object - ``__dict__`` and weakrefs with less bookkeeping, + :attr:`~object.__dict__` and weakrefs with less bookkeeping, using less memory and with faster access. * API for performing calls using @@ -2006,7 +2007,7 @@ Porting to Python 3.12 internal-only field directly. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, + :py:meth:`~type.__subclasses__` (using :c:func:`PyObject_CallMethod`, for example). * Add support of more formatting options (left aligning, octals, uppercase @@ -2025,7 +2026,7 @@ Porting to Python 3.12 :c:func:`PyUnicode_FromFormatV`. (Contributed by Philip Georgi in :gh:`95504`.) -* Extension classes wanting to add a ``__dict__`` or weak reference slot +* Extension classes wanting to add a :attr:`~object.__dict__` or weak reference slot should use :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5640759e79b734..f9e74a9b8ff9c6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -46,7 +46,7 @@ when researching a change. This article explains the new features in Python 3.13, compared to 3.12. -Python 3.13 will be released on October 1, 2024. +Python 3.13 was released on October 7, 2024. For full details, see the :ref:`changelog `. .. seealso:: @@ -60,7 +60,7 @@ Summary -- Release Highlights .. This section singles out the most important changes in Python 3.13. Brevity is key. -Python 3.13 will be the latest stable release of the Python programming +Python 3.13 is the latest stable release of the Python programming language, with a mix of changes to the language, the implementation and the standard library. The biggest changes include a new `interactive interpreter @@ -121,9 +121,10 @@ Interpreter improvements: Python data model improvements: -* :attr:`~class.__static_attributes__` stores the names of attributes accessed +* :attr:`~type.__static_attributes__` stores the names of attributes accessed through ``self.X`` in any function in a class body. -* :attr:`!__firstlineno__` records the first line number of a class definition. +* :attr:`~type.__firstlineno__` records the first line number of a class + definition. Significant improvements in the standard library: @@ -319,12 +320,9 @@ The free-threaded mode requires a different executable, usually called ``python3.13t`` or ``python3.13t.exe``. Pre-built binaries marked as *free-threaded* can be installed as part of the official :ref:`Windows ` -and :ref:`macOS ` installers, +and :ref:`macOS ` installers, or CPython can be built from source with the :option:`--disable-gil` option. -.. better macOS link pending - https://github.com/python/cpython/issues/109975#issuecomment-2286391179 - Free-threaded execution allows for full utilization of the available processing power by running threads in parallel on available CPU cores. While not all software will benefit from this automatically, programs @@ -500,30 +498,6 @@ are not tier 3 supported platforms, but will have best-effort support. .. seealso:: :pep:`730`, :pep:`738` -.. _whatsnew313-incremental-gc: - -Incremental garbage collection ------------------------------- - -The cycle garbage collector is now incremental. -This means that maximum pause times are reduced -by an order of magnitude or more for larger heaps. - -There are now only two generations: young and old. -When :func:`gc.collect` is not called directly, the -GC is invoked a little less frequently. When invoked, it -collects the young generation and an increment of the -old generation, instead of collecting one or more generations. - -The behavior of :func:`!gc.collect` changes slightly: - -* ``gc.collect(1)``: Performs an increment of garbage collection, - rather than collecting generation 1. -* Other calls to :func:`!gc.collect` are unchanged. - -(Contributed by Mark Shannon in :gh:`108362`.) - - Other Language Changes ====================== @@ -588,7 +562,7 @@ Other Language Changes (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in :gh:`73965`.) -* Classes have a new :attr:`~class.__static_attributes__` attribute. +* Classes have a new :attr:`~type.__static_attributes__` attribute. This is populated by the compiler with a tuple of the class's attribute names which are assigned through ``self.`` from any function in its body. (Contributed by Irit Katriel in :gh:`115775`.) @@ -823,6 +797,28 @@ copy (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* As a consequence of necessary internal refactoring, initialization of + internal metaclasses now happens in ``__init__`` rather + than in ``__new__``. This affects projects that subclass these internal + metaclasses to provide custom initialization. + Generally: + + - Custom logic that was done in ``__new__`` after calling ``super().__new__`` + should be moved to ``__init__``. + - To create a class, call the metaclass, not only the metaclass's + ``__new__`` method. + + See :gh:`124520` for discussion and links to changes in some affected + projects. + +* :class:`ctypes.Structure` objects have a new :attr:`~ctypes.Structure._align_` + attribute which allows the alignment of the structure being packed to/from + memory to be specified explicitly. + (Contributed by Matt Sanderson in :gh:`112433`) + dbm --- @@ -902,36 +898,6 @@ fractions (Contributed by Mark Dickinson in :gh:`111320`.) -gc --- - -The cyclic garbage collector is now incremental, -which changes the meaning of the results of -:meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` -as well as :meth:`~gc.get_count` and :meth:`~gc.get_stats`. - -* For backwards compatibility, :meth:`~gc.get_threshold` continues to return - a three-item tuple. - The first value is the threshold for young collections, as before; - the second value determines the rate at which the old collection is scanned - (the default is 10, and higher values mean that the old collection - is scanned more slowly). - The third value is meaningless and is always zero. - -* :meth:`~gc.set_threshold` ignores any items after the second. - -* :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return - the same format of results. - The only difference is that instead of the results referring to - the young, aging and old generations, - the results refer to the young generation - and the aging and collecting spaces of the old generation. - -In summary, code that attempted to manipulate the behavior of the cycle GC -may not work exactly as intended, but it is very unlikely to be harmful. -All other code will work just fine. - - glob ---- @@ -956,12 +922,12 @@ importlib * :func:`~importlib.resources.read_text` These functions are no longer deprecated and are not scheduled for removal. - (Contributed by Petr Viktorin in :gh:`106532`.) + (Contributed by Petr Viktorin in :gh:`116608`.) * :func:`~importlib.resources.contents` remains deprecated in favor of the fully-featured :class:`~importlib.resources.abc.Traversable` API. However, there is now no plan to remove it. - (Contributed by Petr Viktorin in :gh:`106532`.) + (Contributed by Petr Viktorin in :gh:`116608`.) io @@ -1496,11 +1462,6 @@ zipimport Optimizations ============= -* The new :ref:`incremental garbage collector ` - means that maximum pause times are reduced - by an order of magnitude or more for larger heaps. - (Contributed by Mark Shannon in :gh:`108362`.) - * Several standard library modules have had their import times significantly improved. For example, the import time of the :mod:`typing` module @@ -1602,7 +1563,7 @@ and are now removed: * :mod:`!msilib` * :mod:`!nis` * :mod:`!nntplib`: - Use the :pypi:`nntplib` library from PyPI instead. + Use the :pypi:`pynntp` library from PyPI instead. * :mod:`!ossaudiodev`: For audio playback, use the :pypi:`pygame` library from PyPI instead. * :mod:`!pipes`: @@ -2191,13 +2152,13 @@ New Features * Add the :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully qualified name. - The module name is prepended if ``type.__module__`` is a string - and is not equal to either ``'builtins'`` or ``'__main__'``. + The module name is prepended if :attr:`type.__module__` is + a string and is not equal to either ``'builtins'`` or ``'__main__'``. (Contributed by Victor Stinner in :gh:`111696`.) * Add the :c:func:`PyType_GetModuleName` function - to get the type's module name. - This is equivalent to getting the ``type.__module__`` attribute. + to get the type's module name. This is equivalent to getting the + :attr:`type.__module__` attribute. (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) * Add the :c:func:`PyUnicode_EqualToUTF8AndSize` @@ -2534,9 +2495,9 @@ Build Changes * Building CPython now requires a compiler with support for the C11 atomic library, GCC built-in atomic functions, or MSVC interlocked intrinsics. -* Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate +* Autoconf 2.71 and aclocal 1.16.5 are now required to regenerate the :file:`configure` script. - (Contributed by Christian Heimes in :gh:`89886`.) + (Contributed by Christian Heimes in :gh:`89886` and by Victor Stinner in :gh:`112090`.) * SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. @@ -2613,13 +2574,6 @@ Changes in the Python API Wrap it in :func:`staticmethod` if you want to preserve the old behavior. (Contributed by Serhiy Storchaka in :gh:`121027`.) -* The :ref:`garbage collector is now incremental `, - which means that the behavior of :func:`gc.collect` changes slightly: - - * ``gc.collect(1)``: Performs an increment of garbage collection, - rather than collecting generation 1. - * Other calls to :func:`!gc.collect` are unchanged. - * An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to retrieve a username, instead of :exc:`ImportError` on non-Unix platforms diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5acb9bfe18b2d0..b106578fe9e8b0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1,6 +1,6 @@ **************************** - What's New In Python 3.14 + What's new in Python 3.14 **************************** :Editor: TBD @@ -56,7 +56,7 @@ For full details, see the :ref:`changelog `. so it's worth checking back even after reading earlier versions. -Summary -- Release highlights +Summary -- release highlights ============================= .. This section singles out the most important changes in Python 3.14. @@ -67,12 +67,12 @@ Summary -- Release highlights -New Features +New features ============ -.. _whatsnew-314-pep649: +.. _whatsnew314-pep649: -PEP 649: Deferred Evaluation of Annotations +PEP 649: deferred evaluation of annotations ------------------------------------------- The :term:`annotations ` on functions, classes, and modules are no @@ -91,7 +91,7 @@ annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VA format (which evaluates annotations to runtime values, similar to the behavior in earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format (which replaces undefined names with special markers), and the -:attr:`~annotationlib.Format.SOURCE` format (which returns annotations as strings). +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). This example shows how these formats behave: @@ -106,7 +106,7 @@ This example shows how these formats behave: NameError: name 'Undefined' is not defined >>> get_annotations(func, format=Format.FORWARDREF) {'arg': ForwardRef('Undefined')} - >>> get_annotations(func, format=Format.SOURCE) + >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} Implications for annotated code @@ -150,12 +150,12 @@ In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` directive, which turns all annotations into strings. This directive is now considered deprecated and it is expected to be removed in a future version of Python. However, this removal will not happen until after Python 3.13, the last version of -Python without deferred evaluation of annotations, reaches its end of life. +Python without deferred evaluation of annotations, reaches its end of life in 2029. In Python 3.14, the behavior of code using ``from __future__ import annotations`` is unchanged. -Improved Error Messages +Improved error messages ----------------------- * When unpacking assignment fails due to incorrect number of variables, the @@ -172,26 +172,26 @@ Improved Error Messages ValueError: too many values to unpack (expected 3, got 4) -Other Language Changes +Other language changes ====================== * Incorrect usage of :keyword:`await` and asynchronous comprehensions is now detected even if the code is optimized away by the :option:`-O` - command line option. For example, ``python -O -c 'assert await 1'`` + command-line option. For example, ``python -O -c 'assert await 1'`` now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) * Writes to ``__debug__`` are now detected even if the code is optimized - away by the :option:`-O` command line option. For example, + away by the :option:`-O` command-line option. For example, ``python -O -c 'assert (__debug__ := 1)'`` now produces a :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) -* Added class methods :meth:`float.from_number` and :meth:`complex.from_number` +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` to convert a number to :class:`float` or :class:`complex` type correspondingly. They raise an error if the argument is a string. (Contributed by Serhiy Storchaka in :gh:`84978`.) -New Modules +New modules =========== * :mod:`annotationlib`: For introspecting :term:`annotations `. @@ -199,15 +199,22 @@ New Modules (Contributed by Jelle Zijlstra in :gh:`119180`.) -Improved Modules +Improved modules ================ +argparse +-------- + +* The default value of the :ref:`program name ` for + :class:`argparse.ArgumentParser` now reflects the way the Python + interpreter was instructed to find the ``__main__`` module code. + (Contributed by Serhiy Storchaka and Alyssa Coghlan in :gh:`66436`.) ast --- -* Added :func:`ast.compare` for comparing two ASTs. - (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) +* Add :func:`ast.compare` for comparing two ASTs. + (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) * Add support for :func:`copy.replace` for AST nodes. (Contributed by Bénédikt Tran in :gh:`121141`.) @@ -215,6 +222,9 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) +* The ``repr()`` output for AST nodes now includes more information. + (Contributed by Tomas R in :gh:`116022`.) + ctypes ------ @@ -229,56 +239,94 @@ ctypes to help match a non-default ABI. (Contributed by Petr Viktorin in :gh:`97702`.) +decimal +------- + +* Add alternative :class:`~decimal.Decimal` constructor + :meth:`Decimal.from_number() `. + (Contributed by Serhiy Storchaka in :gh:`121798`.) + +datetime +-------- + +* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. + (Contributed by Wannes Boeykens in :gh:`41431`.) dis --- -* Added support for rendering full source location information of +* Add support for rendering full source location information of :class:`instructions `, rather than only the line number. - This feature is added to the following interfaces via the ``show_positions`` + This feature is added to the following interfaces via the *show_positions* keyword argument: - - :class:`dis.Bytecode`, - - :func:`dis.dis`, :func:`dis.distb`, and - - :func:`dis.disassemble`. + - :class:`dis.Bytecode` + - :func:`dis.dis` + - :func:`dis.distb` + - :func:`dis.disassemble` This feature is also exposed via :option:`dis --show-positions`. - (Contributed by Bénédikt Tran in :gh:`123165`.) fractions --------- -Added support for converting any objects that have the -:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. -(Contributed by Serhiy Storchaka in :gh:`82017`.) +* Add support for converting any objects that have the + :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + (Contributed by Serhiy Storchaka in :gh:`82017`.) + +* Add alternative :class:`~fractions.Fraction` constructor + :meth:`Fraction.from_number() `. + (Contributed by Serhiy Storchaka in :gh:`121797`.) + + +functools +--------- + +* Add support to :func:`functools.partial` and + :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels + to reserve a place for positional arguments. + (Contributed by Dominykas Grigonis in :gh:`119127`.) http ---- -Directory lists and error pages generated by the :mod:`http.server` -module allow the browser to apply its default dark mode. -(Contributed by Yorik Hansen in :gh:`123430`.) +* Directory lists and error pages generated by the :mod:`http.server` + module allow the browser to apply its default dark mode. + (Contributed by Yorik Hansen in :gh:`123430`.) + + +inspect +------- + +* :func:`inspect.signature` takes a new argument *annotation_format* to control + the :class:`annotationlib.Format` used for representing annotations. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + +* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without surrounding quotes. + (Contributed by Jelle Zijlstra in :gh:`101552`.) json ---- -Add notes for JSON serialization errors that allow to identify the source -of the error. -(Contributed by Serhiy Storchaka in :gh:`122163`.) +* Add notes for JSON serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122163`.) -Enable :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. -See the :ref:`JSON command-line interface ` documentation. -(Contributed by Trey Hunner in :gh:`122873`.) +* Enable the :mod:`json` module to work as a script using the :option:`-m` + switch: :program:`python -m json`. + See the :ref:`JSON command-line interface ` documentation. + (Contributed by Trey Hunner in :gh:`122873`.) operator -------- -* Two new functions ``operator.is_none`` and ``operator.is_not_none`` +* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` have been added, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. @@ -288,7 +336,7 @@ operator os -- -* Added the :data:`os.environ.refresh() ` method to update +* Add the :data:`os.environ.refresh() ` method to update :data:`os.environ` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) @@ -311,32 +359,43 @@ pathlib pdb --- -* Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. As a result, all the instance specific data like :pdbcmd:`display` and - :pdbcmd:`commands` are preserved across hard-coded breakpoints. + :pdbcmd:`commands` are preserved across hardcoded breakpoints. (Contributed by Tian Gao in :gh:`121450`.) +* Add a new argument *mode* to :class:`pdb.Pdb`. Disable the ``restart`` + command when :mod:`pdb` is in ``inline`` mode. + (Contributed by Tian Gao in :gh:`123757`.) pickle ------ * Set the default protocol version on the :mod:`pickle` module to 5. - For more details, please see :ref:`pickle protocols `. + For more details, see :ref:`pickle protocols `. * Add notes for pickle serialization errors that allow to identify the source of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) +pydoc +----- + +* :term:`Annotations ` in help output are now usually + displayed in a format closer to that in the original source. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + + symtable -------- * Expose the following :class:`symtable.Symbol` methods: - * :meth:`~symtable.Symbol.is_free_class` - * :meth:`~symtable.Symbol.is_comp_iter` * :meth:`~symtable.Symbol.is_comp_cell` + * :meth:`~symtable.Symbol.is_comp_iter` + * :meth:`~symtable.Symbol.is_free_class` (Contributed by Bénédikt Tran in :gh:`120029`.) @@ -361,12 +420,26 @@ asyncio Deprecated ========== +* :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`builtins`: Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) +* :mod:`multiprocessing` and :mod:`concurrent.futures`: + The default start method (see :ref:`multiprocessing-start-methods`) changed + away from *fork* to *forkserver* on platforms where it was not already + *spawn* (Windows & macOS). If you require the threading incompatible *fork* + start method you must explicitly specify it when using :mod:`multiprocessing` + or :mod:`concurrent.futures` APIs. + (Contributed by Gregory P. Smith in :gh:`84559`.) + * :mod:`os`: :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to @@ -401,17 +474,17 @@ ast * Remove the following classes. They were all deprecated since Python 3.8, and have emitted deprecation warnings since Python 3.12: - * :class:`!ast.Num` - * :class:`!ast.Str` * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` * :class:`!ast.Ellipsis` + * :class:`!ast.NameConstant` + * :class:`!ast.Num` + * :class:`!ast.Str` Use :class:`ast.Constant` instead. As a consequence of these removals, user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom :class:`ast.NodeVisitor` subclasses will no longer be called when the - ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` method instead. Also, remove the following deprecated properties on :class:`ast.Constant`, @@ -430,16 +503,16 @@ asyncio * Remove the following classes and functions. They were all deprecated and emitted deprecation warnings since Python 3.12: + * :func:`!asyncio.get_child_watcher` + * :func:`!asyncio.set_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :class:`!asyncio.MultiLoopChildWatcher` * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.ThreadedChildWatcher` + * :class:`!asyncio.MultiLoopChildWatcher` * :class:`!asyncio.PidfdChildWatcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :func:`!asyncio.get_child_watcher` - * :func:`!asyncio.set_child_watcher` + * :class:`!asyncio.SafeChildWatcher` + * :class:`!asyncio.ThreadedChildWatcher` (Contributed by Kumar Aditya in :gh:`120804`.) @@ -546,14 +619,20 @@ Changes in the Python API Wrap it in :func:`staticmethod` if you want to preserve the old behavior. (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) -Build Changes +* The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` + locale in some cases. + This temporary change affects other threads. + (Contributed by Serhiy Storchaka in :gh:`69998`.) + + +Build changes ============= -C API Changes +C API changes ============= -New Features +New features ------------ * Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. @@ -562,18 +641,18 @@ New Features * Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` object: - * :c:func:`PyUnicodeWriter_Create`. - * :c:func:`PyUnicodeWriter_Discard`. - * :c:func:`PyUnicodeWriter_Finish`. - * :c:func:`PyUnicodeWriter_WriteChar`. - * :c:func:`PyUnicodeWriter_WriteUTF8`. - * :c:func:`PyUnicodeWriter_WriteUCS4`. - * :c:func:`PyUnicodeWriter_WriteWideChar`. - * :c:func:`PyUnicodeWriter_WriteStr`. - * :c:func:`PyUnicodeWriter_WriteRepr`. - * :c:func:`PyUnicodeWriter_WriteSubstring`. - * :c:func:`PyUnicodeWriter_Format`. - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` (Contributed by Victor Stinner in :gh:`119182`.) @@ -585,23 +664,23 @@ New Features is backwards incompatible to any C-Extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will - normally result in crashes during the exectuion of the subsequent call to + normally result in crashes during the execution of the subsequent call to :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use an address sanitizer to identify any use-after-free coming from an interned string and deallocate it during module shutdown. - (Contribued by Eddie Elizondo in :gh:`113601`.) + (Contributed by Eddie Elizondo in :gh:`113601`.) * Add new functions to convert C ```` numbers from/to Python :class:`int`: - * :c:func:`PyLong_FromInt32` - * :c:func:`PyLong_FromInt64` - * :c:func:`PyLong_FromUInt32` - * :c:func:`PyLong_FromUInt64` * :c:func:`PyLong_AsInt32` * :c:func:`PyLong_AsInt64` * :c:func:`PyLong_AsUInt32` * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` (Contributed by Victor Stinner in :gh:`120389`.) @@ -624,20 +703,20 @@ New Features * Add functions to configure the Python initialization (:pep:`741`): + * :c:func:`Py_InitializeFromInitConfig` + * :c:func:`PyInitConfig_AddModule` * :c:func:`PyInitConfig_Create` * :c:func:`PyInitConfig_Free` + * :c:func:`PyInitConfig_FreeStrList` * :c:func:`PyInitConfig_GetError` * :c:func:`PyInitConfig_GetExitCode` - * :c:func:`PyInitConfig_HasOption` * :c:func:`PyInitConfig_GetInt` * :c:func:`PyInitConfig_GetStr` * :c:func:`PyInitConfig_GetStrList` - * :c:func:`PyInitConfig_FreeStrList` + * :c:func:`PyInitConfig_HasOption` * :c:func:`PyInitConfig_SetInt` * :c:func:`PyInitConfig_SetStr` * :c:func:`PyInitConfig_SetStrList` - * :c:func:`PyInitConfig_AddModule` - * :c:func:`Py_InitializeFromInitConfig` (Contributed by Victor Stinner in :gh:`107954`.) @@ -646,6 +725,10 @@ New Features `__ mentioned in :pep:`630` (:gh:`124153`). +* Add :c:func:`PyUnicode_Equal` function to the limited C API: + test if two strings are equal. + (Contributed by Victor Stinner in :gh:`124502`.) + Porting to Python 3.14 ---------------------- @@ -665,12 +748,7 @@ Deprecated :c:macro:`!isfinite` available from :file:`math.h` since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) -* :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - -.. Add deprecations above alphabetically, not here at the end. +.. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index c09fa839886305..7104904c956a7a 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -312,8 +312,8 @@ cluttering source directories, the *pyc* files are now collected in a Aside from the filenames and target directories, the new scheme has a few aspects that are visible to the programmer: -* Imported modules now have a :attr:`__cached__` attribute which stores the name - of the actual file that was imported: +* Imported modules now have a :attr:`~module.__cached__` attribute which stores + the name of the actual file that was imported: >>> import collections >>> collections.__cached__ # doctest: +SKIP diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 95b89e7579fcce..f814c4e90d5719 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -549,9 +549,11 @@ separation of binary and text data). PEP 3155: Qualified name for classes and functions ================================================== -Functions and class objects have a new ``__qualname__`` attribute representing +Functions and class objects have a new :attr:`~definition.__qualname__` +attribute representing the "path" from the module top-level to their definition. For global functions -and classes, this is the same as ``__name__``. For other functions and classes, +and classes, this is the same as :attr:`~definition.__name__`. +For other functions and classes, it provides better information about where they were actually defined, and how they might be accessible from the global scope. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 71425120c37185..9d746b378995c3 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2271,7 +2271,8 @@ Changes in the Python API :func:`super` and falling through all the way to the ABCs. For compatibility, catch both :exc:`NotImplementedError` or the appropriate exception as needed. -* The module type now initializes the :attr:`__package__` and :attr:`__loader__` +* The module type now initializes the :attr:`~module.__package__` and + :attr:`~module.__loader__` attributes to ``None`` by default. To determine if these attributes were set in a backwards-compatible fashion, use e.g. ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 077d8c1aae91ae..3f3f634171dab6 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -423,8 +423,8 @@ are declared in the annotations:: return 'Hello ' + name While these annotations are available at runtime through the usual -:attr:`__annotations__` attribute, *no automatic type checking happens at -runtime*. Instead, it is assumed that a separate off-line type checker +:attr:`~object.__annotations__` attribute, *no automatic type checking happens +at runtime*. Instead, it is assumed that a separate off-line type checker (e.g. `mypy `_) will be used for on-demand source code analysis. @@ -2526,9 +2526,9 @@ Changes in the C API to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) -* Because the lack of the :attr:`__module__` attribute breaks pickling and +* Because the lack of the :attr:`~type.__module__` attribute breaks pickling and introspection, a deprecation warning is now raised for builtin types without - the :attr:`__module__` attribute. This would be an AttributeError in + the :attr:`~type.__module__` attribute. This will be an :exc:`AttributeError` in the future. (Contributed by Serhiy Storchaka in :issue:`20204`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index be83aa8a8550c5..2276fed60c8db3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -549,7 +549,7 @@ PEP 520: Preserving Class Attribute Definition Order Attributes in a class definition body have a natural ordering: the same order in which the names appear in the source. This order is now -preserved in the new class's :attr:`~object.__dict__` attribute. +preserved in the new class's :attr:`~type.__dict__` attribute. Also, the effective default class *execution* namespace (returned from :ref:`type.__prepare__() `) is now an insertion-order-preserving @@ -934,7 +934,7 @@ asynchronous generators. The :func:`~collections.namedtuple` function now accepts an optional keyword argument *module*, which, when specified, is used for -the ``__module__`` attribute of the returned named tuple class. +the :attr:`~type.__module__` attribute of the returned named tuple class. (Contributed by Raymond Hettinger in :issue:`17941`.) The *verbose* and *rename* arguments for diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index d0e60bc280a217..fc9f49e65af847 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -428,9 +428,9 @@ Other Language Changes normal assignment syntax:: >>> def parse(family): - lastname, *members = family.split() - return lastname.upper(), *members - + ... lastname, *members = family.split() + ... return lastname.upper(), *members + ... >>> parse('simpsons homer marge bart lisa maggie') ('SIMPSONS', 'homer', 'marge', 'bart', 'lisa', 'maggie') diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 1b24abfa857150..6118b02dd9bd48 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -637,7 +637,8 @@ pydoc ----- The documentation string is now shown not only for class, function, -method etc, but for any object that has its own ``__doc__`` attribute. +method etc, but for any object that has its own :attr:`~definition.__doc__` +attribute. (Contributed by Serhiy Storchaka in :issue:`40257`.) random diff --git a/Grammar/python.gram b/Grammar/python.gram index e9a8c69c4fa27c..b47028460b94f4 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -151,9 +151,9 @@ assignment[stmt_ty]: | a=('(' b=single_target ')' { b } | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { CHECK_VERSION(stmt_ty, 6, "Variable annotations syntax is", _PyAST_AnnAssign(a, b, c, 0, EXTRA)) } - | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] { + | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=annotated_rhs !'=' tc=[TYPE_COMMENT] { _PyAST_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | a=single_target b=augassign ~ c=(yield_expr | star_expressions) { + | a=single_target b=augassign ~ c=annotated_rhs { _PyAST_AugAssign(a, b->kind, c, EXTRA) } | invalid_assignment diff --git a/Include/Python.h b/Include/Python.h index 8fffa22df9da48..e1abdd16f031fb 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -55,6 +55,10 @@ # include // __readgsqword() #endif +#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) +# include // __readgsqword() +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 58d93fcfc1066b..03622698113ee7 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,6 +8,8 @@ extern "C" { #endif +/* Total tool ids available */ +#define _PY_MONITORING_TOOL_IDS 8 /* Count of all local monitoring events */ #define _PY_MONITORING_LOCAL_EVENTS 10 /* Count of all "real" monitoring events (not derived from other events) */ @@ -57,6 +59,8 @@ typedef struct { _Py_LocalMonitors active_monitors; /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; + /* The version of tools when they instrument the code */ + uintptr_t tool_versions[_PY_MONITORING_TOOL_IDS]; /* Information to support line events */ _PyCoLineInstrumentationData *lines; /* The tools that are to be notified for line events for the matching code unit */ diff --git a/Include/cpython/context.h b/Include/cpython/context.h index a509f4eaba3d77..3c9be7873b9399 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -28,21 +28,31 @@ PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); typedef enum { - Py_CONTEXT_EVENT_ENTER, - Py_CONTEXT_EVENT_EXIT, + /* + * A context has been entered, causing the "current context" to switch to + * it. The object passed to the watch callback is the now-current + * contextvars.Context object. Each enter event will eventually have a + * corresponding exit event for the same context object after any + * subsequently entered contexts have themselves been exited. + */ + Py_CONTEXT_EVENT_ENTER, + /* + * A context is about to be exited, which will cause the "current context" + * to switch back to what it was before the context was entered. The + * object passed to the watch callback is the still-current + * contextvars.Context object. + */ + Py_CONTEXT_EVENT_EXIT, } PyContextEvent; /* - * A Callback to clue in non-python contexts impls about a - * change in the active python context. - * - * The callback is invoked with the event and a reference to = - * the context after its entered and before its exited. + * Context object watcher callback function. The object passed to the callback + * is event-specific; see PyContextEvent for details. * * if the callback returns with an exception set, it must return -1. Otherwise * it should return 0 */ -typedef int (*PyContext_WatchCallback)(PyContextEvent, PyContext *); +typedef int (*PyContext_WatchCallback)(PyContextEvent, PyObject *); /* * Register a per-interpreter callback that will be invoked for context object diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index e2861c963266ea..b113c7fdcf6515 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -14,16 +14,11 @@ typedef struct { /* Number of items in the dictionary */ Py_ssize_t ma_used; - /* Dictionary version: globally unique, value change each time - the dictionary is modified */ -#ifdef Py_BUILD_CORE - /* Bits 0-7 are for dict watchers. + /* This is a private field for CPython's internal use. + * Bits 0-7 are for dict watchers. * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) - * The remaining bits (12-63) are the actual version tag. */ - uint64_t ma_version_tag; -#else - Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif + * The remaining bits are not currently used. */ + uint64_t _ma_watcher_tag; PyDictKeysObject *ma_keys; diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 82f8cc8a159c77..c1214d5e3714ea 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,6 +2,9 @@ # error "this header file must not be included directly" #endif +#define _PyLong_CAST(op) \ + (assert(PyLong_Check(op)), _Py_CAST(PyLongObject*, (op))) + PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); #define Py_ASNATIVEBYTES_DEFAULTS -1 @@ -71,10 +74,9 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); absolute value of a long. For example, this returns 1 for 1 and -1, 2 for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. v must not be NULL, and must be a normalized long. - (uint64_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. + Always successful. */ -PyAPI_FUNC(uint64_t) _PyLong_NumBits(PyObject *v); +PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v); /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 28acf7265a0856..9aa1a92c413fe9 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -42,13 +42,13 @@ struct _PyWeakReference { PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); +#define _PyWeakref_CAST(op) \ + (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { - PyWeakReference *ref; - PyObject *obj; - assert(PyWeakref_Check(ref_obj)); - ref = _Py_CAST(PyWeakReference*, ref_obj); - obj = ref->wr_object; + PyWeakReference *ref = _PyWeakref_CAST(ref_obj); + PyObject *obj = ref->wr_object; // Explanation for the Py_REFCNT() check: when a weakref's target is part // of a long chain of deallocations which triggers the trashcan mechanism, // clearing the weakrefs can be delayed long after the target's refcount diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3db3aa3eb77879..20436a68b69677 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -15,13 +15,7 @@ extern "C" { typedef struct { - union { - struct { - uint16_t backoff : 4; - uint16_t value : 12; - }; - uint16_t as_counter; // For printf("%#x", ...) - }; + uint16_t value_and_backoff; } _Py_BackoffCounter; @@ -38,17 +32,19 @@ typedef struct { and a 4-bit 'backoff' field. When resetting the counter, the backoff field is incremented (until it reaches a limit) and the value is set to a bit mask representing the value 2**backoff - 1. - The maximum backoff is 12 (the number of value bits). + The maximum backoff is 12 (the number of bits in the value). There is an exceptional value which must not be updated, 0xFFFF. */ -#define UNREACHABLE_BACKOFF 0xFFFF +#define BACKOFF_BITS 4 +#define MAX_BACKOFF 12 +#define UNREACHABLE_BACKOFF 15 static inline bool is_unreachable_backoff_counter(_Py_BackoffCounter counter) { - return counter.as_counter == UNREACHABLE_BACKOFF; + return counter.value_and_backoff == UNREACHABLE_BACKOFF; } static inline _Py_BackoffCounter @@ -57,8 +53,7 @@ make_backoff_counter(uint16_t value, uint16_t backoff) assert(backoff <= 15); assert(value <= 0xFFF); _Py_BackoffCounter result; - result.value = value; - result.backoff = backoff; + result.value_and_backoff = (value << BACKOFF_BITS) | backoff; return result; } @@ -66,7 +61,7 @@ static inline _Py_BackoffCounter forge_backoff_counter(uint16_t counter) { _Py_BackoffCounter result; - result.as_counter = counter; + result.value_and_backoff = counter; return result; } @@ -74,41 +69,42 @@ static inline _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter counter) { assert(!is_unreachable_backoff_counter(counter)); - if (counter.backoff < 12) { - return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1); + int backoff = counter.value_and_backoff & 15; + if (backoff < MAX_BACKOFF) { + return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); } else { - return make_backoff_counter((1 << 12) - 1, 12); + return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); } } static inline _Py_BackoffCounter pause_backoff_counter(_Py_BackoffCounter counter) { - return make_backoff_counter(counter.value | 1, counter.backoff); + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff | (1 << BACKOFF_BITS); + return result; } static inline _Py_BackoffCounter advance_backoff_counter(_Py_BackoffCounter counter) { - if (!is_unreachable_backoff_counter(counter)) { - return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff); - } - else { - return counter; - } + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff - (1 << BACKOFF_BITS); + return result; } static inline bool backoff_counter_triggers(_Py_BackoffCounter counter) { - return counter.value == 0; + /* Test whether the value is zero and the backoff is not UNREACHABLE_BACKOFF */ + return counter.value_and_backoff < UNREACHABLE_BACKOFF; } /* Initial JUMP_BACKWARD counter. * This determines when we create a trace for a loop. * Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ -#define JUMP_BACKWARD_INITIAL_VALUE 16 +#define JUMP_BACKWARD_INITIAL_VALUE 15 #define JUMP_BACKWARD_INITIAL_BACKOFF 4 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -122,7 +118,7 @@ initial_jump_backoff_counter(void) * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ -#define SIDE_EXIT_INITIAL_VALUE 64 +#define SIDE_EXIT_INITIAL_VALUE 63 #define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter @@ -136,7 +132,7 @@ initial_temperature_backoff_counter(void) static inline _Py_BackoffCounter initial_unreachable_backoff_counter(void) { - return forge_backoff_counter(UNREACHABLE_BACKOFF); + return make_backoff_counter(0, UNREACHABLE_BACKOFF); } #ifdef __cplusplus diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a97b53028c8f59..cff2b1f7114793 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -251,6 +251,7 @@ typedef struct _special_method { } _Py_SpecialMethod; PyAPI_DATA(const _Py_SpecialMethod) _Py_SpecialMethods[]; +PyAPI_DATA(const size_t) _Py_FunctionAttributeOffsets[]; PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); @@ -274,6 +275,8 @@ PyAPI_FUNC(PyObject *) _PyEval_GetANext(PyObject *aiter); PyAPI_FUNC(void) _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto); PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); +PyAPI_FUNC(int) +_Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); /* Bits that can be set in PyThreadState.eval_breaker */ #define _PY_GIL_DROP_REQUEST_BIT (1U << 0) @@ -283,6 +286,7 @@ PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFra #define _PY_GC_SCHEDULED_BIT (1U << 4) #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) +#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 @@ -312,6 +316,8 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); +PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 5e2d5c5ce9d868..4400be8b33dee7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -21,6 +21,17 @@ extern void _PyCodec_Fini(PyInterpreterState *interp); extern PyObject* _PyCodec_Lookup(const char *encoding); +/* + * Un-register the error handling callback function registered under + * the given 'name'. Only custom error handlers can be un-registered. + * + * - Return -1 and set an exception if 'name' refers to a built-in + * error handling name (e.g., 'strict'), or if an error occurred. + * - Return 0 if no custom error handler can be found for 'name'. + * - Return 1 if the custom error handler was successfully removed. + */ +extern int _PyCodec_UnregisterError(const char *name); + /* Text codec specific encoding and decoding API. Checks the encoding against a list of codecs which do not diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index f9a043b0208c8f..1920724c1d4f57 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -230,31 +230,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) -#ifdef Py_GIL_DISABLED - -#define THREAD_LOCAL_DICT_VERSION_COUNT 256 -#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT - -static inline uint64_t -dict_next_version(PyInterpreterState *interp) -{ - PyThreadState *tstate = PyThreadState_GET(); - uint64_t cur_progress = (tstate->dict_global_version & - (THREAD_LOCAL_DICT_VERSION_BATCH - 1)); - if (cur_progress == 0) { - uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version, - THREAD_LOCAL_DICT_VERSION_BATCH); - tstate->dict_global_version = next; - } - return tstate->dict_global_version += DICT_VERSION_INCREMENT; -} - -#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP) - -#else -#define DICT_NEXT_VERSION(INTERP) \ - ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) -#endif PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, @@ -263,7 +238,7 @@ _PyDict_SendEvent(int watcher_bits, PyObject *key, PyObject *value); -static inline uint64_t +static inline void _PyDict_NotifyEvent(PyInterpreterState *interp, PyDict_WatchEvent event, PyDictObject *mp, @@ -271,12 +246,11 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; + int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); } - return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 1a44755c7a01a3..11932b8d1e1ab6 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -12,10 +12,6 @@ extern "C" { #define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; uint32_t next_keys_version; PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a77658134fae8c..c9ac3819d0390b 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -62,7 +62,7 @@ enum _frameowner { typedef struct _PyInterpreterFrame { _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ struct _PyInterpreterFrame *previous; - PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ + _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ @@ -84,6 +84,12 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + assert(PyFunction_Check(func)); + return (PyFunctionObject *)func; +} + static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } @@ -144,14 +150,15 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * */ static inline void _PyFrame_Initialize( - _PyInterpreterFrame *frame, PyFunctionObject *func, + _PyInterpreterFrame *frame, _PyStackRef func, PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { frame->previous = previous; - frame->f_funcobj = (PyObject *)func; + frame->f_funcobj = func; frame->f_executable = PyStackRef_FromPyObjectNew(code); - frame->f_builtins = func->func_builtins; - frame->f_globals = func->func_globals; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + frame->f_builtins = func_obj->func_builtins; + frame->f_globals = func_obj->func_globals; frame->f_locals = locals; frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; @@ -300,10 +307,11 @@ PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFr * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from, _PyInterpreterFrame * previous) +_PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); - PyCodeObject *code = (PyCodeObject *)func->func_code; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + PyCodeObject *code = (PyCodeObject *)func_obj->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); @@ -321,7 +329,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); frame->previous = previous; - frame->f_funcobj = Py_None; + frame->f_funcobj = PyStackRef_None; frame->f_executable = PyStackRef_FromPyObjectNew(code); #ifdef Py_DEBUG frame->f_builtins = NULL; @@ -345,7 +353,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int } PyAPI_FUNC(_PyInterpreterFrame *) -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous); diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 762c583ce94e9a..4e04cf431e0b31 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -20,6 +20,7 @@ extern "C" { # define Py_async_gen_asends_MAXFREELIST 80 # define Py_futureiters_MAXFREELIST 255 # define Py_object_stack_chunks_MAXFREELIST 4 +# define Py_unicode_writers_MAXFREELIST 1 // A generic freelist of either PyObjects or other data structures. struct _Py_freelist { @@ -44,6 +45,7 @@ struct _Py_freelists { struct _Py_freelist async_gen_asends; struct _Py_freelist futureiters; struct _Py_freelist object_stack_chunks; + struct _Py_freelist unicode_writers; }; #ifdef __cplusplus diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 6d44e933e8a8cb..c45d281125febb 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -18,6 +18,10 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 +#define FUNC_VERSION_UNSET 0 +#define FUNC_VERSION_CLEARED 1 +#define FUNC_VERSION_FIRST_VALID 2 + #define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct _func_version_cache_item { @@ -41,6 +45,12 @@ struct _py_func_state { extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); +static inline int +_PyFunction_IsVersionValid(uint32_t version) +{ + return version >= FUNC_VERSION_FIRST_VALID; +} + extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); void _PyFunction_ClearCodeByVersion(uint32_t version); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index cb67a7ee2b3402..cf96f661e6cd7e 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -387,6 +387,17 @@ union _PyStackRef; extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); +// Like Py_VISIT but for _PyStackRef fields +#define _Py_VISIT_STACKREF(ref) \ + do { \ + if (!PyStackRef_IsNull(ref)) { \ + int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 913dce6f1ec0fe..e3f7ac707f0c37 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -66,9 +66,6 @@ struct _Py_static_objects { struct _Py_interp_cached_objects { PyObject *interned_strings; - /* AST */ - PyObject *str_replace_inf; - /* object.__reduce__ */ PyObject *objreduce; PyObject *type_slots_pname; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6e948e16b7dbe8..2fd7d5d13a98b2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -11,7 +11,7 @@ extern "C" { #ifdef Py_DEBUG static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { - if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { + if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); _PyObject_Dump(obj); } @@ -562,6 +562,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); @@ -604,7 +605,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classdictcell__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__complex__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__contains__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__copy__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ctypes_from_outparam__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__del__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__delattr__)); @@ -758,6 +758,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_isatty_open_only)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); @@ -769,7 +770,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_date)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_datetime)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 5c63a6e519b93d..fc3871570cc49d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -48,6 +48,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; @@ -93,7 +94,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__classdictcell__) STRUCT_FOR_ID(__complex__) STRUCT_FOR_ID(__contains__) - STRUCT_FOR_ID(__copy__) STRUCT_FOR_ID(__ctypes_from_outparam__) STRUCT_FOR_ID(__del__) STRUCT_FOR_ID(__delattr__) @@ -247,6 +247,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) + STRUCT_FOR_ID(_isatty_open_only) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) @@ -258,7 +259,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) STRUCT_FOR_ID(_strptime) - STRUCT_FOR_ID(_strptime_datetime) + STRUCT_FOR_ID(_strptime_datetime_date) + STRUCT_FOR_ID(_strptime_datetime_datetime) + STRUCT_FOR_ID(_strptime_datetime_time) STRUCT_FOR_ID(_type_) STRUCT_FOR_ID(_uninitialized_submodules) STRUCT_FOR_ID(_warn_unawaited_coroutine) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 36366429e8db25..36cd71e5a007d5 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -35,7 +35,7 @@ extern "C" { #include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeid.h" // struct _Py_type_id_pool +#include "pycore_uniqueid.h" // struct _Py_unique_id_pool #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -102,9 +102,8 @@ struct _is { PyInterpreterState *next; int64_t id; - int64_t id_refcount; + Py_ssize_t id_refcount; int requires_idref; - PyThread_type_lock id_mutex; #define _PyInterpreterState_WHENCE_NOTSET -1 #define _PyInterpreterState_WHENCE_UNKNOWN 0 @@ -221,7 +220,7 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state - struct _Py_type_id_pool type_ids; + struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; #endif @@ -261,7 +260,7 @@ struct _is { struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - + size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; @@ -272,6 +271,7 @@ struct _is { Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + uintptr_t monitoring_tool_versions[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; @@ -317,8 +317,7 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst PyAPI_FUNC(int64_t) _PyInterpreterState_ObjectToID(PyObject *); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpIDObject(PyObject *); -PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); -PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsReady(PyInterpreterState *interp); diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index e6da083b807ce5..57cbce8f126aca 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -64,8 +64,8 @@ PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) } } -// Unlock a mutex, returns 0 if the mutex is not locked (used for improved -// error messages). +// Unlock a mutex, returns -1 if the mutex is not locked (used for improved +// error messages) otherwise returns 0. extern int _PyMutex_TryUnlock(PyMutex *m); @@ -160,8 +160,9 @@ typedef struct { PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m); PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m); +extern PyLockStatus _PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags); PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m); - +extern int _PyRecursiveMutex_TryUnlock(_PyRecursiveMutex *m); // A readers-writer (RW) lock. The lock supports multiple concurrent readers or // a single writer. The lock is write-preferring: if a writer is waiting while diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 8822147b636dd4..196b4152280a35 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -79,11 +79,10 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) } // _PyLong_Frexp returns a double x and an exponent e such that the -// true value is approximately equal to x * 2**e. e is >= 0. x is +// true value is approximately equal to x * 2**e. x is // 0.0 if and only if the input is 0 (in which case, e and x are both -// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is -// possible if the number of bits doesn't fit into a Py_ssize_t, sets -// OverflowError and returns -1.0 for x, 0 for e. +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. +// Always successful. // // Export for 'math' shared extension PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e); @@ -105,10 +104,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 095eb0f8a89b79..a88ff2deeba941 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -258,6 +258,8 @@ Known values: Python 3.14a1 3604 (Do not duplicate test at end of while statements) Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) Python 3.14a1 3606 (Specialize CALL_KW) + Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) + Python 3.14a1 3608 (Add support for slices) Python 3.15 will start with 3650 @@ -270,7 +272,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3606 +#define PYC_MAGIC_NUMBER 3608 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 049677b292e235..cc2dda48ed9f28 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -46,7 +46,7 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) { } PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); -PyObject* _Py_module_getattro(PyModuleObject *m, PyObject *name); +PyObject* _Py_module_getattro(PyObject *m, PyObject *name); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 80b588815bc9cf..8832692d03c29e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,10 +14,7 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_typeid.h" // _PyType_IncrefSlow - - -#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) +#include "pycore_uniqueid.h" // _PyType_IncrefSlow // This value is added to `ob_ref_shared` for objects that use deferred // reference counting so that they are not immediately deallocated when the @@ -27,25 +24,8 @@ extern "C" { // `ob_ref_shared` are used for flags. #define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) -// gh-121528, gh-118997: Similar to _Py_IsImmortal() but be more loose when -// comparing the reference count to stay compatible with C extensions built -// with the stable ABI 3.11 or older. Such extensions implement INCREF/DECREF -// as refcnt++ and refcnt-- without taking in account immortal objects. For -// example, the reference count of an immortal object can change from -// _Py_IMMORTAL_REFCNT to _Py_IMMORTAL_REFCNT+1 (INCREF) or -// _Py_IMMORTAL_REFCNT-1 (DECREF). -// -// This function should only be used in assertions. Otherwise, _Py_IsImmortal() -// must be used instead. -static inline int _Py_IsImmortalLoose(PyObject *op) -{ -#if defined(Py_GIL_DISABLED) - return _Py_IsImmortal(op); -#else - return (op->ob_refcnt >= _Py_IMMORTAL_REFCNT_LOOSE); -#endif -} -#define _Py_IsImmortalLoose(op) _Py_IsImmortalLoose(_PyObject_CAST(op)) +/* For backwards compatibility -- Do not use this */ +#define _Py_IsImmortalLoose(op) _Py_IsImmortal /* Check if an object is consistent. For example, ensure that the reference @@ -97,7 +77,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); #else #define _PyObject_HEAD_INIT(type) \ { \ - .ob_refcnt = _Py_IMMORTAL_REFCNT, \ + .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ .ob_type = (type) \ } #endif @@ -184,7 +164,7 @@ PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) { if (op) { - assert(_Py_IsImmortalLoose(op)); + assert(_Py_IsImmortal(op)); #ifdef Py_GIL_DISABLED op->ob_tid = _Py_UNOWNED_TID; op->ob_ref_local = 0; @@ -316,7 +296,7 @@ static inline void _Py_INCREF_TYPE(PyTypeObject *type) { if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - assert(_Py_IsImmortalLoose(type)); + assert(_Py_IsImmortal(type)); _Py_INCREF_IMMORTAL_STAT_INC(); return; } @@ -335,12 +315,12 @@ _Py_INCREF_TYPE(PyTypeObject *type) // Unsigned comparison so that `unique_id=-1`, which indicates that // per-thread refcounting has been disabled on this type, is handled by // the "else". - if ((size_t)ht->unique_id < (size_t)tstate->types.size) { + if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) { # ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); # endif _Py_INCREF_STAT_INC(); - tstate->types.refcounts[ht->unique_id]++; + tstate->refcounts.values[ht->unique_id]++; } else { // The slow path resizes the thread-local refcount array if necessary. @@ -357,7 +337,7 @@ static inline void _Py_DECREF_TYPE(PyTypeObject *type) { if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - assert(_Py_IsImmortalLoose(type)); + assert(_Py_IsImmortal(type)); _Py_DECREF_IMMORTAL_STAT_INC(); return; } @@ -368,12 +348,12 @@ _Py_DECREF_TYPE(PyTypeObject *type) // Unsigned comparison so that `unique_id=-1`, which indicates that // per-thread refcounting has been disabled on this type, is handled by // the "else". - if ((size_t)ht->unique_id < (size_t)tstate->types.size) { + if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) { # ifdef Py_REF_DEBUG _Py_DECREF_DecRefTotal(); # endif _Py_DECREF_STAT_INC(); - tstate->types.refcounts[ht->unique_id]--; + tstate->refcounts.values[ht->unique_id]--; } else { // Directly decref the type if the type id is not assigned or if @@ -393,7 +373,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortalLoose(typeobj)); + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); _Py_INCREF_TYPE(typeobj); _Py_NewReference(op); } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 51479afae3833d..c18423476d3962 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -22,6 +22,8 @@ extern "C" { ((OP) == STORE_FAST_MAYBE_NULL) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ + ((OP) == JUMP_IF_FALSE) || \ + ((OP) == JUMP_IF_TRUE) || \ ((OP) == SETUP_FINALLY) || \ ((OP) == SETUP_CLEANUP) || \ ((OP) == SETUP_WITH) || \ @@ -269,6 +271,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; case JUMP_NO_INTERRUPT: return 0; case LIST_APPEND: @@ -477,8 +483,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 5; case YIELD_VALUE: return 1; - case _DO_CALL_FUNCTION_EX: - return 3 + (oparg & 1); default: return -1; } @@ -726,6 +730,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; case JUMP_NO_INTERRUPT: return 0; case LIST_APPEND: @@ -934,8 +942,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 6; case YIELD_VALUE: return 1; - case _DO_CALL_FUNCTION_EX: - return 1; default: return -1; } @@ -956,7 +962,7 @@ enum InstructionFormat { }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < 264) && \ + (((OP) >= 0) && ((OP) < 266) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -1005,22 +1011,22 @@ struct opcode_metadata { int16_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[264]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { +const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, @@ -1069,7 +1075,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1104,8 +1110,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,10 +1123,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, 0 }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1130,7 +1136,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1138,7 +1144,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1161,7 +1167,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1182,13 +1188,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, @@ -1208,10 +1214,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1221,9 +1227,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, - [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, @@ -1355,8 +1362,8 @@ _PyOpcode_macro_expansion[256] = { [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION_PUSH_KEYS, 1, 2 }, { _LOAD_GLOBAL_BUILTINS_FROM_KEYS, 1, 3 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION_PUSH_KEYS, 1, 1 }, { _LOAD_GLOBAL_MODULE_FROM_KEYS, 1, 3 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, 0, 0 } } }, @@ -1422,9 +1429,9 @@ _PyOpcode_macro_expansion[256] = { }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[264]; +extern const char *_PyOpcode_OpName[266]; #ifdef NEED_OPCODE_METADATA -const char *_PyOpcode_OpName[264] = { +const char *_PyOpcode_OpName[266] = { [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -1543,6 +1550,8 @@ const char *_PyOpcode_OpName[264] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_IF_FALSE] = "JUMP_IF_FALSE", + [JUMP_IF_TRUE] = "JUMP_IF_TRUE", [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", @@ -1647,7 +1656,6 @@ const char *_PyOpcode_OpName[264] = { [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [YIELD_VALUE] = "YIELD_VALUE", - [_DO_CALL_FUNCTION_EX] = "_DO_CALL_FUNCTION_EX", }; #endif @@ -1894,12 +1902,12 @@ const uint8_t _PyOpcode_Deopt[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, [WITH_EXCEPT_START] = WITH_EXCEPT_START, [YIELD_VALUE] = YIELD_VALUE, - [_DO_CALL_FUNCTION_EX] = _DO_CALL_FUNCTION_EX, }; #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 116: \ case 117: \ case 118: \ case 119: \ @@ -1943,25 +1951,28 @@ const uint8_t _PyOpcode_Deopt[256] = { case 235: \ ; struct pseudo_targets { - uint8_t targets[3]; + uint8_t as_sequence; + uint8_t targets[4]; }; -extern const struct pseudo_targets _PyOpcode_PseudoTargets[8]; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[10]; #ifdef NEED_OPCODE_METADATA -const struct pseudo_targets _PyOpcode_PseudoTargets[8] = { - [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, - [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, - [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, - [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, - [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, - [SETUP_CLEANUP-256] = { { NOP, 0, 0 } }, - [SETUP_WITH-256] = { { NOP, 0, 0 } }, - [POP_BLOCK-256] = { { NOP, 0, 0 } }, +const struct pseudo_targets _PyOpcode_PseudoTargets[10] = { + [LOAD_CLOSURE-256] = { 0, { LOAD_FAST, 0, 0, 0 } }, + [STORE_FAST_MAYBE_NULL-256] = { 0, { STORE_FAST, 0, 0, 0 } }, + [JUMP-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD, 0, 0 } }, + [JUMP_NO_INTERRUPT-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0, 0 } }, + [JUMP_IF_FALSE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_FALSE, 0 } }, + [JUMP_IF_TRUE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_TRUE, 0 } }, + [SETUP_FINALLY-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_CLEANUP-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_WITH-256] = { 0, { NOP, 0, 0, 0 } }, + [POP_BLOCK-256] = { 0, { NOP, 0, 0, 0 } }, }; #endif // NEED_OPCODE_METADATA static inline bool is_pseudo_target(int pseudo, int target) { - if (pseudo < 256 || pseudo >= 264) { + if (pseudo < 256 || pseudo >= 266) { return false; } for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 19e54bf122a8bb..f92c0a0cddf906 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -29,9 +29,10 @@ typedef struct { typedef struct { uint8_t opcode; uint8_t oparg; - uint16_t valid:1; - uint16_t linked:1; - uint16_t chain_depth:14; // Must be big engough for MAX_CHAIN_DEPTH - 1. + uint8_t valid:1; + uint8_t linked:1; + uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool warm; int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; @@ -123,11 +124,18 @@ PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #ifdef _Py_TIER2 PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); + #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) +# define _Py_Executors_InvalidateCold(A) ((void)0) + #endif +// Used as the threshold to trigger executor invalidation when +// trace_run_counter is greater than this value. +#define JIT_CLEANUP_THRESHOLD 100000 // This is the length of the trace we project initially. #define UOP_MAX_TRACE_LENGTH 800 diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index f3f5942444e851..a1e084cf67d58d 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -152,6 +152,19 @@ PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); * a non-zero value on failure. */ PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); +/* + * Hangs the thread indefinitely without exiting it. + * + * gh-87135: There is no safe way to exit a thread other than returning + * normally from its start function. This is used during finalization in lieu + * of actually exiting the thread. Since the program is expected to terminate + * soon anyway, it does not matter if the thread stack stays around until then. + * + * This is unfortunate for embedders who may not be terminating their process + * when they're done with the interpreter, but our C API design does not allow + * for safely exiting threads attempting to re-enter Python post finalization. + */ +void _Py_NO_RETURN PyThread_hang_thread(void); #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e6adb98eb19130..a17ba46966daa1 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_function.h" #include "pycore_object.h" // _PyObject_HEAD_INIT #include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT #include "pycore_parser.h" // _parser_runtime_state_INIT @@ -243,7 +244,7 @@ extern PyTypeObject _PyExc_MemoryError; .dict_state = _dict_state_INIT, \ .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \ .func_state = { \ - .next_version = 1, \ + .next_version = FUNC_VERSION_FIRST_VALID, \ }, \ .types = { \ .next_version_tag = _Py_TYPE_BASE_VERSION_TAG, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index bac6b5b8fcfd9d..3b80e265b0ca50 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -557,6 +557,7 @@ extern "C" { INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } @@ -602,7 +603,6 @@ extern "C" { INIT_ID(__classdictcell__), \ INIT_ID(__complex__), \ INIT_ID(__contains__), \ - INIT_ID(__copy__), \ INIT_ID(__ctypes_from_outparam__), \ INIT_ID(__del__), \ INIT_ID(__delattr__), \ @@ -756,6 +756,7 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ + INIT_ID(_isatty_open_only), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ @@ -767,7 +768,9 @@ extern "C" { INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ INIT_ID(_strptime), \ - INIT_ID(_strptime_datetime), \ + INIT_ID(_strptime_datetime_date), \ + INIT_ID(_strptime_datetime_datetime), \ + INIT_ID(_strptime_datetime_time), \ INIT_ID(_type_), \ INIT_ID(_uninitialized_submodules), \ INIT_ID(_warn_unawaited_coroutine), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index b5b6993812057d..588e57f6cd97e0 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -60,90 +60,50 @@ typedef union _PyStackRef { #define Py_TAG_BITS ((uintptr_t)1) #ifdef Py_GIL_DISABLED - static const _PyStackRef PyStackRef_NULL = { .bits = 0 | Py_TAG_DEFERRED}; -#else - static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; -#endif +static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; #define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) +#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) +#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) +#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) - -#ifdef Py_GIL_DISABLED -# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) -#else -# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) }) -#endif - -#ifdef Py_GIL_DISABLED -# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) -#else -# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) }) -#endif - -#ifdef Py_GIL_DISABLED -# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) -#else -# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) }) -#endif - -// Note: the following are all macros because MSVC (Windows) has trouble inlining them. - -#define PyStackRef_Is(a, b) ((a).bits == (b).bits) - -#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) - - -#ifdef Py_GIL_DISABLED -// Gets a PyObject * from a _PyStackRef static inline PyObject * PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) { PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); return cleared; } -#else -# define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits) -#endif -// Converts a PyStackRef back to a PyObject *, stealing the -// PyStackRef. -#ifdef Py_GIL_DISABLED +#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) + +static inline PyObject * +PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) +{ + assert(!PyStackRef_IsDeferred(stackref)); + return (PyObject *)stackref.bits; +} + static inline PyObject * PyStackRef_AsPyObjectSteal(_PyStackRef stackref) { - if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) { + assert(!PyStackRef_IsNull(stackref)); + if (PyStackRef_IsDeferred(stackref)) { return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); } return PyStackRef_AsPyObjectBorrow(stackref); } -#else -# define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref) -#endif -// Converts a PyStackRef back to a PyObject *, converting the -// stackref to a new reference. -#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) - -#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) - -// Converts a PyObject * to a PyStackRef, stealing the reference -#ifdef Py_GIL_DISABLED static inline _PyStackRef _PyStackRef_FromPyObjectSteal(PyObject *obj) { + assert(obj != NULL); // Make sure we don't take an already tagged value. assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - unsigned int tag = (obj == NULL || _Py_IsImmortal(obj)) ? (Py_TAG_DEFERRED) : Py_TAG_PTR; + unsigned int tag = _Py_IsImmortal(obj) ? (Py_TAG_DEFERRED) : Py_TAG_PTR; return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); } # define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) -#else -# define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) -#endif - -// Converts a PyObject * to a PyStackRef, with a new reference -#ifdef Py_GIL_DISABLED static inline _PyStackRef PyStackRef_FromPyObjectNew(PyObject *obj) { @@ -157,13 +117,8 @@ PyStackRef_FromPyObjectNew(PyObject *obj) return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; } } -# define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) -#else -# define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) }) -#endif +#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) -#ifdef Py_GIL_DISABLED -// Same as PyStackRef_FromPyObjectNew but only for immortal objects. static inline _PyStackRef PyStackRef_FromPyObjectImmortal(PyObject *obj) { @@ -173,59 +128,30 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj) assert(_Py_IsImmortal(obj)); return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; } -# define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) -#else -# define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) }) -#endif +#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) - -#define PyStackRef_CLEAR(op) \ - do { \ - _PyStackRef *_tmp_op_ptr = &(op); \ - _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ - if (!PyStackRef_IsNull(_tmp_old_op)) { \ - *_tmp_op_ptr = PyStackRef_NULL; \ - PyStackRef_CLOSE(_tmp_old_op); \ - } \ - } while (0) - -#ifdef Py_GIL_DISABLED -# define PyStackRef_CLOSE(REF) \ +#define PyStackRef_CLOSE(REF) \ do { \ _PyStackRef _close_tmp = (REF); \ + assert(!PyStackRef_IsNull(_close_tmp)); \ if (!PyStackRef_IsDeferred(_close_tmp)) { \ Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ } \ } while (0) -#else -# define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)) -#endif -#define PyStackRef_XCLOSE(stackref) \ - do { \ - _PyStackRef _tmp = (stackref); \ - if (!PyStackRef_IsNull(_tmp)) { \ - PyStackRef_CLOSE(_tmp); \ - } \ - } while (0); - - -#ifdef Py_GIL_DISABLED static inline _PyStackRef PyStackRef_DUP(_PyStackRef stackref) { + assert(!PyStackRef_IsNull(stackref)); if (PyStackRef_IsDeferred(stackref)) { - assert(PyStackRef_IsNull(stackref) || - _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || - _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref))); + assert(_Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || + _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref)) + ); return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); return stackref; } -#else -# define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))) -#endif // Convert a possibly deferred reference to a strong reference. static inline _PyStackRef @@ -234,13 +160,65 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); } -static inline void -_PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length) -{ - for (size_t i = 0; i < length; i++) { - dst[i] = PyStackRef_AsPyObjectBorrow(src[i]); - } -} +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref) + + +#else // Py_GIL_DISABLED + +// With GIL +static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; +#define PyStackRef_IsNull(stackref) ((stackref).bits == 0) +#define PyStackRef_True ((_PyStackRef){.bits = (uintptr_t)&_Py_TrueStruct }) +#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) }) +#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) }) + +#define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits) + +#define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref) + +#define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) + +#define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) }) + +#define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) }) + +#define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))) + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) _Py_DECREF_SPECIALIZED(PyStackRef_AsPyObjectBorrow(stackref), dealloc) + +#endif // Py_GIL_DISABLED + +// Note: this is a macro because MSVC (Windows) has trouble inlining it. + +#define PyStackRef_Is(a, b) ((a).bits == (b).bits) + +// Converts a PyStackRef back to a PyObject *, converting the +// stackref to a new reference. +#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (!PyStackRef_IsNull(_tmp_old_op)) { \ + *_tmp_op_ptr = PyStackRef_NULL; \ + PyStackRef_CLOSE(_tmp_old_op); \ + } \ + } while (0) + +#define PyStackRef_XCLOSE(stackref) \ + do { \ + _PyStackRef _tmp = (stackref); \ + if (!PyStackRef_IsNull(_tmp)) { \ + PyStackRef_CLOSE(_tmp); \ + } \ + } while (0); + + // StackRef type checks diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index f681b644c9ad5d..a72ef4493b77ca 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -32,15 +32,15 @@ typedef struct _PyThreadStateImpl { struct _Py_freelists freelists; struct _brc_thread_state brc; struct { - // The thread-local refcounts for heap type objects - Py_ssize_t *refcounts; + // The per-thread refcounts + Py_ssize_t *values; // Size of the refcounts array. Py_ssize_t size; - // If set, don't use thread-local refcounts + // If set, don't use per-thread refcounts int is_finalized; - } types; + } refcounts; #endif #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) diff --git a/Include/internal/pycore_typeid.h b/Include/internal/pycore_typeid.h deleted file mode 100644 index e64d1447f6b51d..00000000000000 --- a/Include/internal/pycore_typeid.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef Py_INTERNAL_TYPEID_H -#define Py_INTERNAL_TYPEID_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#ifdef Py_GIL_DISABLED - -// This contains code for allocating unique ids to heap type objects -// and re-using those ids when the type is deallocated. -// -// The type ids are used to implement per-thread reference counts of -// heap type objects to avoid contention on the reference count fields -// of heap type objects. Static type objects are immortal, so contention -// is not an issue for those types. -// -// Type id of -1 is used to indicate a type doesn't use thread-local -// refcounting. This value is used when a type object is finalized by the GC -// and during interpreter shutdown to allow the type object to be -// deallocated promptly when the object's refcount reaches zero. -// -// Each entry implicitly represents a type id based on it's offset in the -// table. Non-allocated entries form a free-list via the 'next' pointer. -// Allocated entries store the corresponding PyTypeObject. -typedef union _Py_type_id_entry { - // Points to the next free type id, when part of the freelist - union _Py_type_id_entry *next; - - // Stores the type object when the id is assigned - PyHeapTypeObject *type; -} _Py_type_id_entry; - -struct _Py_type_id_pool { - PyMutex mutex; - - // combined table of types with allocated type ids and unallocated - // type ids. - _Py_type_id_entry *table; - - // Next entry to allocate inside 'table' or NULL - _Py_type_id_entry *freelist; - - // size of 'table' - Py_ssize_t size; -}; - -// Assigns the next id from the pool of type ids. -extern void _PyType_AssignId(PyHeapTypeObject *type); - -// Releases the allocated type id back to the pool. -extern void _PyType_ReleaseId(PyHeapTypeObject *type); - -// Merges the thread-local reference counts into the corresponding types. -extern void _PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate); - -// Like _PyType_MergeThreadLocalRefcounts, but also frees the thread-local -// array of refcounts. -extern void _PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate); - -// Frees the interpreter's pool of type ids. -extern void _PyType_FinalizeIdPool(PyInterpreterState *interp); - -// Increfs the type, resizing the thread-local refcount array if necessary. -PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); - -#endif /* Py_GIL_DISABLED */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_TYPEID_H */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index ca5a1e2adb4787..118bc98b35d5e3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -209,7 +209,6 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *); // Export for _testinternalcapi extension. PyAPI_FUNC(PyObject *) _PyType_GetSlotWrapperNames(void); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 20497ee93016d0..a60372f58295a9 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -252,11 +252,7 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); -/* Fast equality check when the inputs are known to be exact unicode types - and where the hash values are equal (i.e. a very probable match) */ -extern int _PyUnicode_EQ(PyObject *, PyObject *); - -// Equality check. +// Fast equality check when the inputs are known to be exact unicode types. // Export for '_pickle' shared extension. PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index efdbde4c8ea3c6..eb2eca06ec4d4f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -172,10 +172,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(__copy__); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ctypes_from_outparam__); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -788,6 +784,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_isatty_open_only); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_length_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -832,7 +832,15 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_strptime_datetime); + string = &_Py_ID(_strptime_datetime_date); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_datetime); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_time); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); @@ -2928,6 +2936,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(str_replace_inf); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uniqueid.h b/Include/internal/pycore_uniqueid.h new file mode 100644 index 00000000000000..8f3b4418408cf8 --- /dev/null +++ b/Include/internal/pycore_uniqueid.h @@ -0,0 +1,72 @@ +#ifndef Py_INTERNAL_UNIQUEID_H +#define Py_INTERNAL_UNIQUEID_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique ids to objects for per-thread +// reference counting. +// +// Per-thread reference counting is used along with deferred reference +// counting to avoid scaling bottlenecks due to reference count contention. +// +// An id of -1 is used to indicate that an object doesn't use per-thread +// refcounting. This value is used when the object is finalized by the GC +// and during interpreter shutdown to allow the object to be +// deallocated promptly when the object's refcount reaches zero. +// +// Each entry implicitly represents a unique id based on its offset in the +// table. Non-allocated entries form a free-list via the 'next' pointer. +// Allocated entries store the corresponding PyObject. +typedef union _Py_unique_id_entry { + // Points to the next free type id, when part of the freelist + union _Py_unique_id_entry *next; + + // Stores the object when the id is assigned + PyObject *obj; +} _Py_unique_id_entry; + +struct _Py_unique_id_pool { + PyMutex mutex; + + // combined table of object with allocated unique ids and unallocated ids. + _Py_unique_id_entry *table; + + // Next entry to allocate inside 'table' or NULL + _Py_unique_id_entry *freelist; + + // size of 'table' + Py_ssize_t size; +}; + +// Assigns the next id from the pool of ids. +extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj); + +// Releases the allocated id back to the pool. +extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id); + +// Merges the per-thread reference counts into the corresponding objects. +extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread +// array of refcounts. +extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Frees the interpreter's pool of type ids. +extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp); + +// Increfs the type, resizing the per-thread refcount array if necessary. +PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNIQUEID_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b950f760d74ac7..1951c65a2871cf 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -97,54 +97,56 @@ extern "C" { #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE #define _DO_CALL 356 -#define _DO_CALL_KW 357 -#define _DYNAMIC_EXIT 358 +#define _DO_CALL_FUNCTION_EX 357 +#define _DO_CALL_KW 358 +#define _DYNAMIC_EXIT 359 #define _END_SEND END_SEND -#define _ERROR_POP_N 359 +#define _ERROR_POP_N 360 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 360 -#define _EXPAND_METHOD_KW 361 -#define _FATAL_ERROR 362 +#define _EXPAND_METHOD 361 +#define _EXPAND_METHOD_KW 362 +#define _FATAL_ERROR 363 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 363 -#define _FOR_ITER_GEN_FRAME 364 -#define _FOR_ITER_TIER_TWO 365 +#define _FOR_ITER 364 +#define _FOR_ITER_GEN_FRAME 365 +#define _FOR_ITER_TIER_TWO 366 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 366 -#define _GUARD_BOTH_INT 367 -#define _GUARD_BOTH_UNICODE 368 -#define _GUARD_BUILTINS_VERSION 369 -#define _GUARD_DORV_NO_DICT 370 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 371 -#define _GUARD_GLOBALS_VERSION 372 -#define _GUARD_IS_FALSE_POP 373 -#define _GUARD_IS_NONE_POP 374 -#define _GUARD_IS_NOT_NONE_POP 375 -#define _GUARD_IS_TRUE_POP 376 -#define _GUARD_KEYS_VERSION 377 -#define _GUARD_NOS_FLOAT 378 -#define _GUARD_NOS_INT 379 -#define _GUARD_NOT_EXHAUSTED_LIST 380 -#define _GUARD_NOT_EXHAUSTED_RANGE 381 -#define _GUARD_NOT_EXHAUSTED_TUPLE 382 -#define _GUARD_TOS_FLOAT 383 -#define _GUARD_TOS_INT 384 -#define _GUARD_TYPE_VERSION 385 +#define _GUARD_BOTH_FLOAT 367 +#define _GUARD_BOTH_INT 368 +#define _GUARD_BOTH_UNICODE 369 +#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 370 +#define _GUARD_DORV_NO_DICT 371 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 372 +#define _GUARD_GLOBALS_VERSION 373 +#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 374 +#define _GUARD_IS_FALSE_POP 375 +#define _GUARD_IS_NONE_POP 376 +#define _GUARD_IS_NOT_NONE_POP 377 +#define _GUARD_IS_TRUE_POP 378 +#define _GUARD_KEYS_VERSION 379 +#define _GUARD_NOS_FLOAT 380 +#define _GUARD_NOS_INT 381 +#define _GUARD_NOT_EXHAUSTED_LIST 382 +#define _GUARD_NOT_EXHAUSTED_RANGE 383 +#define _GUARD_NOT_EXHAUSTED_TUPLE 384 +#define _GUARD_TOS_FLOAT 385 +#define _GUARD_TOS_INT 386 +#define _GUARD_TYPE_VERSION 387 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 386 -#define _INIT_CALL_PY_EXACT_ARGS 387 -#define _INIT_CALL_PY_EXACT_ARGS_0 388 -#define _INIT_CALL_PY_EXACT_ARGS_1 389 -#define _INIT_CALL_PY_EXACT_ARGS_2 390 -#define _INIT_CALL_PY_EXACT_ARGS_3 391 -#define _INIT_CALL_PY_EXACT_ARGS_4 392 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 388 +#define _INIT_CALL_PY_EXACT_ARGS 389 +#define _INIT_CALL_PY_EXACT_ARGS_0 390 +#define _INIT_CALL_PY_EXACT_ARGS_1 391 +#define _INIT_CALL_PY_EXACT_ARGS_2 392 +#define _INIT_CALL_PY_EXACT_ARGS_3 393 +#define _INIT_CALL_PY_EXACT_ARGS_4 394 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -156,130 +158,135 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 393 -#define _IS_NONE 394 +#define _INTERNAL_INCREMENT_OPT_COUNTER 395 +#define _IS_NONE 396 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 395 -#define _ITER_CHECK_RANGE 396 -#define _ITER_CHECK_TUPLE 397 -#define _ITER_JUMP_LIST 398 -#define _ITER_JUMP_RANGE 399 -#define _ITER_JUMP_TUPLE 400 -#define _ITER_NEXT_LIST 401 -#define _ITER_NEXT_RANGE 402 -#define _ITER_NEXT_TUPLE 403 -#define _JUMP_TO_TOP 404 +#define _ITER_CHECK_LIST 397 +#define _ITER_CHECK_RANGE 398 +#define _ITER_CHECK_TUPLE 399 +#define _ITER_JUMP_LIST 400 +#define _ITER_JUMP_RANGE 401 +#define _ITER_JUMP_TUPLE 402 +#define _ITER_NEXT_LIST 403 +#define _ITER_NEXT_RANGE 404 +#define _ITER_NEXT_TUPLE 405 +#define _JUMP_TO_TOP 406 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 405 -#define _LOAD_ATTR_CLASS 406 -#define _LOAD_ATTR_CLASS_0 407 -#define _LOAD_ATTR_CLASS_1 408 +#define _LOAD_ATTR 407 +#define _LOAD_ATTR_CLASS 408 +#define _LOAD_ATTR_CLASS_0 409 +#define _LOAD_ATTR_CLASS_1 410 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 409 -#define _LOAD_ATTR_INSTANCE_VALUE_0 410 -#define _LOAD_ATTR_INSTANCE_VALUE_1 411 -#define _LOAD_ATTR_METHOD_LAZY_DICT 412 -#define _LOAD_ATTR_METHOD_NO_DICT 413 -#define _LOAD_ATTR_METHOD_WITH_VALUES 414 -#define _LOAD_ATTR_MODULE 415 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 416 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 417 -#define _LOAD_ATTR_PROPERTY_FRAME 418 -#define _LOAD_ATTR_SLOT 419 -#define _LOAD_ATTR_SLOT_0 420 -#define _LOAD_ATTR_SLOT_1 421 -#define _LOAD_ATTR_WITH_HINT 422 +#define _LOAD_ATTR_INSTANCE_VALUE 411 +#define _LOAD_ATTR_INSTANCE_VALUE_0 412 +#define _LOAD_ATTR_INSTANCE_VALUE_1 413 +#define _LOAD_ATTR_METHOD_LAZY_DICT 414 +#define _LOAD_ATTR_METHOD_NO_DICT 415 +#define _LOAD_ATTR_METHOD_WITH_VALUES 416 +#define _LOAD_ATTR_MODULE 417 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 418 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 419 +#define _LOAD_ATTR_PROPERTY_FRAME 420 +#define _LOAD_ATTR_SLOT 421 +#define _LOAD_ATTR_SLOT_0 422 +#define _LOAD_ATTR_SLOT_1 423 +#define _LOAD_ATTR_WITH_HINT 424 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 423 -#define _LOAD_CONST_INLINE_BORROW 424 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 425 -#define _LOAD_CONST_INLINE_WITH_NULL 426 +#define _LOAD_CONST_INLINE 425 +#define _LOAD_CONST_INLINE_BORROW 426 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 427 +#define _LOAD_CONST_INLINE_WITH_NULL 428 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 427 -#define _LOAD_FAST_0 428 -#define _LOAD_FAST_1 429 -#define _LOAD_FAST_2 430 -#define _LOAD_FAST_3 431 -#define _LOAD_FAST_4 432 -#define _LOAD_FAST_5 433 -#define _LOAD_FAST_6 434 -#define _LOAD_FAST_7 435 +#define _LOAD_FAST 429 +#define _LOAD_FAST_0 430 +#define _LOAD_FAST_1 431 +#define _LOAD_FAST_2 432 +#define _LOAD_FAST_3 433 +#define _LOAD_FAST_4 434 +#define _LOAD_FAST_5 435 +#define _LOAD_FAST_6 436 +#define _LOAD_FAST_7 437 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 436 -#define _LOAD_GLOBAL_BUILTINS 437 -#define _LOAD_GLOBAL_MODULE 438 +#define _LOAD_GLOBAL 438 +#define _LOAD_GLOBAL_BUILTINS 439 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 440 +#define _LOAD_GLOBAL_MODULE 441 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 442 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD +#define _MAKE_CALLARGS_A_TUPLE 443 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAKE_WARM 444 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 439 -#define _MONITOR_CALL 440 -#define _MONITOR_JUMP_BACKWARD 441 -#define _MONITOR_RESUME 442 +#define _MAYBE_EXPAND_METHOD 445 +#define _MAYBE_EXPAND_METHOD_KW 446 +#define _MONITOR_CALL 447 +#define _MONITOR_JUMP_BACKWARD 448 +#define _MONITOR_RESUME 449 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 443 -#define _POP_JUMP_IF_TRUE 444 +#define _POP_JUMP_IF_FALSE 450 +#define _POP_JUMP_IF_TRUE 451 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 445 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 452 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 446 +#define _PUSH_FRAME 453 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 447 -#define _PY_FRAME_KW 448 -#define _QUICKEN_RESUME 449 -#define _REPLACE_WITH_TRUE 450 +#define _PY_FRAME_GENERAL 454 +#define _PY_FRAME_KW 455 +#define _QUICKEN_RESUME 456 +#define _REPLACE_WITH_TRUE 457 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 451 -#define _SEND 452 -#define _SEND_GEN_FRAME 453 +#define _SAVE_RETURN_OFFSET 458 +#define _SEND 459 +#define _SEND_GEN_FRAME 460 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 454 -#define _STORE_ATTR 455 -#define _STORE_ATTR_INSTANCE_VALUE 456 -#define _STORE_ATTR_SLOT 457 -#define _STORE_ATTR_WITH_HINT 458 +#define _START_EXECUTOR 461 +#define _STORE_ATTR 462 +#define _STORE_ATTR_INSTANCE_VALUE 463 +#define _STORE_ATTR_SLOT 464 +#define _STORE_ATTR_WITH_HINT 465 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 459 -#define _STORE_FAST_0 460 -#define _STORE_FAST_1 461 -#define _STORE_FAST_2 462 -#define _STORE_FAST_3 463 -#define _STORE_FAST_4 464 -#define _STORE_FAST_5 465 -#define _STORE_FAST_6 466 -#define _STORE_FAST_7 467 +#define _STORE_FAST 466 +#define _STORE_FAST_0 467 +#define _STORE_FAST_1 468 +#define _STORE_FAST_2 469 +#define _STORE_FAST_3 470 +#define _STORE_FAST_4 471 +#define _STORE_FAST_5 472 +#define _STORE_FAST_6 473 +#define _STORE_FAST_7 474 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 468 -#define _STORE_SUBSCR 469 +#define _STORE_SLICE 475 +#define _STORE_SUBSCR 476 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 470 -#define _TO_BOOL 471 +#define _TIER2_RESUME_CHECK 477 +#define _TO_BOOL 478 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -289,14 +296,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 472 +#define _UNPACK_SEQUENCE 479 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 472 +#define MAX_UOP_ID 479 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4d0ab22e6aa8f3..2f0a7fb2f6e549 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -54,10 +54,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, - [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG, [_TO_BOOL_LIST] = HAS_EXIT_FLAG, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_STR] = HAS_EXIT_FLAG, [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, @@ -69,12 +69,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -82,7 +82,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -97,7 +97,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -116,9 +116,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, - [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, - [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_GUARD_GLOBALS_VERSION_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_GUARD_BUILTINS_VERSION_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -156,7 +157,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS_0] = 0, [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, - [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -225,7 +226,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -239,17 +240,19 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD_KW] = HAS_ARG_FLAG, [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CALLARGS_A_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, - [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, @@ -271,9 +274,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, + [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, @@ -394,10 +400,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", - [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_GUARD_BUILTINS_VERSION_PUSH_KEYS] = "_GUARD_BUILTINS_VERSION_PUSH_KEYS", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_PUSH_KEYS] = "_GUARD_GLOBALS_VERSION_PUSH_KEYS", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -473,20 +480,25 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = "_LOAD_GLOBAL_BUILTINS_FROM_KEYS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_FROM_KEYS] = "_LOAD_GLOBAL_MODULE_FROM_KEYS", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_NAME] = "_LOAD_NAME", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_WARM] = "_MAKE_WARM", [_MAP_ADD] = "_MAP_ADD", [_MATCH_CLASS] = "_MATCH_CLASS", [_MATCH_KEYS] = "_MATCH_KEYS", [_MATCH_MAPPING] = "_MATCH_MAPPING", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_TOP] = "_POP_TOP", @@ -746,12 +758,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_GLOBALS_VERSION: return 0; - case _GUARD_BUILTINS_VERSION: + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: return 0; - case _LOAD_GLOBAL_MODULE: - return 0; - case _LOAD_GLOBAL_BUILTINS: + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: return 0; + case _LOAD_GLOBAL_MODULE_FROM_KEYS: + return 1; + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: + return 1; case _DELETE_FAST: return 0; case _MAKE_CELL: @@ -992,6 +1006,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; + case _MAYBE_EXPAND_METHOD_KW: + return 3 + oparg; case _PY_FRAME_KW: return 3 + oparg; case _CHECK_FUNCTION_VERSION_KW: @@ -1004,6 +1020,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 3 + oparg; case _CALL_KW_NON_PY: return 3 + oparg; + case _MAKE_CALLARGS_A_TUPLE: + return 3 + (oparg & 1); case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: @@ -1056,12 +1074,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CHECK_FUNCTION: return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; case _INTERNAL_INCREMENT_OPT_COUNTER: return 1; case _DYNAMIC_EXIT: return 0; case _START_EXECUTOR: return 0; + case _MAKE_WARM: + return 0; case _FATAL_ERROR: return 0; case _CHECK_VALIDITY_AND_SET_IP: diff --git a/Include/object.h b/Include/object.h index 7124f58f6bdb37..5be4dedadc20eb 100644 --- a/Include/object.h +++ b/Include/object.h @@ -81,7 +81,7 @@ whose size is determined when the object is allocated. #else #define PyObject_HEAD_INIT(type) \ { \ - { _Py_IMMORTAL_REFCNT }, \ + { _Py_IMMORTAL_INITIAL_REFCNT }, \ (type) \ }, #endif @@ -180,6 +180,12 @@ _Py_ThreadId(void) tid = __readfsdword(24); #elif defined(_MSC_VER) && defined(_M_ARM64) tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); #elif defined(__i386__) __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 5ded0b41b4830e..327bdb792464a0 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -126,7 +126,6 @@ extern "C" { #define UNPACK_EX 113 #define UNPACK_SEQUENCE 114 #define YIELD_VALUE 115 -#define _DO_CALL_FUNCTION_EX 116 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -226,13 +225,15 @@ extern "C" { #define INSTRUMENTED_LINE 254 #define ENTER_EXECUTOR 255 #define JUMP 256 -#define JUMP_NO_INTERRUPT 257 -#define LOAD_CLOSURE 258 -#define POP_BLOCK 259 -#define SETUP_CLEANUP 260 -#define SETUP_FINALLY 261 -#define SETUP_WITH 262 -#define STORE_FAST_MAYBE_NULL 263 +#define JUMP_IF_FALSE 257 +#define JUMP_IF_TRUE 258 +#define JUMP_NO_INTERRUPT 259 +#define LOAD_CLOSURE 260 +#define POP_BLOCK 261 +#define SETUP_CLEANUP 262 +#define SETUP_FINALLY 263 +#define SETUP_WITH 264 +#define STORE_FAST_MAYBE_NULL 265 #define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 150 diff --git a/Include/py_curses.h b/Include/py_curses.h index 79b1b01fcfa594..e11bfedb17d205 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -81,8 +81,6 @@ typedef struct { char *encoding; } PyCursesWindowObject; -#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) - #define PyCurses_CAPSULE_NAME "_curses._C_API" @@ -99,6 +97,8 @@ static void **PyCurses_API; #define PyCursesInitialised {if (! ((int (*)(void))PyCurses_API[2]) () ) return NULL;} #define PyCursesInitialisedColor {if (! ((int (*)(void))PyCurses_API[3]) () ) return NULL;} +#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) + #define import_curses() \ PyCurses_API = (void **)PyCapsule_Import(PyCurses_CAPSULE_NAME, 1); diff --git a/Include/pythread.h b/Include/pythread.h index a3216c51d66165..82247daf8e0aa0 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -17,7 +17,26 @@ typedef enum PyLockStatus { PyAPI_FUNC(void) PyThread_init_thread(void); PyAPI_FUNC(unsigned long) PyThread_start_new_thread(void (*)(void *), void *); -PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); +/* Terminates the current thread. Considered unsafe. + * + * WARNING: This function is only safe to call if all functions in the full call + * stack are written to safely allow it. Additionally, the behavior is + * platform-dependent. This function should be avoided, and is no longer called + * by Python itself. It is retained only for compatibility with existing C + * extension code. + * + * With pthreads, calls `pthread_exit` causes some libcs (glibc?) to attempt to + * unwind the stack and call C++ destructors; if a `noexcept` function is + * reached, they may terminate the process. Others (macOS) do unwinding. + * + * On Windows, calls `_endthreadex` which kills the thread without calling C++ + * destructors. + * + * In either case there is a risk of invalid references remaining to data on the + * thread stack. + */ +Py_DEPRECATED(3.14) PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); + PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ diff --git a/Include/refcount.h b/Include/refcount.h index 9a4e15065ecab8..141cbd34dd72e6 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -21,25 +21,30 @@ cleanup during runtime finalization. #if SIZEOF_VOID_P > 4 /* -In 64+ bit systems, an object will be marked as immortal by setting all of the -lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF +In 64+ bit systems, any object whose 32 bit reference count is >= 2**31 +will be treated as immortal. Using the lower 32 bits makes the value backwards compatible by allowing C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. +increase and decrease the objects reference count. + +In order to offer sufficient resilience to C extensions using the stable ABI +compiled against 3.11 or earlier, we set the initial value near the +middle of the range (2**31, 2**32). That way the the refcount can be +off by ~1 billion without affecting immortality. Reference count increases will use saturated arithmetic, taking advantage of having all the lower 32 bits set, which will avoid the reference count to go beyond the refcount limit. Immortality checks for reference count decreases will be done by checking the bit sign flag in the lower 32 bits. + */ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3UL << 30)) #else /* -In 32 bit systems, an object will be marked as immortal by setting all of the -lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF +In 32 bit systems, an object will be treated as immortal if its reference +count equals or exceeds _Py_IMMORTAL_MINIMUM_REFCNT (2**30). Using the lower 30 bits makes the value backwards compatible by allowing C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely @@ -47,9 +52,10 @@ increase and decrease the objects reference count. The object would lose its immortality, but the execution would still be correct. Reference count increases and decreases will first go through an immortality -check by comparing the reference count field to the immortality reference count. +check by comparing the reference count field to the minimum immortality refcount. */ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3L << 29)) +#define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30)) #endif // Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is @@ -90,7 +96,7 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #else uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return _Py_IMMORTAL_REFCNT; + return _Py_IMMORTAL_INITIAL_REFCNT; } Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); return _Py_STATIC_CAST(Py_ssize_t, local) + @@ -109,9 +115,9 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 - return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); + return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; #else - return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); + return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; #endif } #define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) @@ -236,7 +242,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) uint32_t new_local = local + 1; if (new_local == 0) { _Py_INCREF_IMMORTAL_STAT_INC(); - // local is equal to _Py_IMMORTAL_REFCNT: do nothing + // local is equal to _Py_IMMORTAL_REFCNT_LOCAL: do nothing return; } if (_Py_IsOwnedByCurrentThread(op)) { @@ -246,18 +252,14 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - // Portable saturated add, branching on the carry flag and set low bits PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; - PY_UINT32_T new_refcnt = cur_refcnt + 1; - if (new_refcnt == 0) { + if (((int32_t)cur_refcnt) < 0) { + // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); - // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, - // do nothing return; } - op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; + op->ob_refcnt_split[PY_BIG_ENDIAN] = cur_refcnt + 1; #else - // Explicitly check immortality against the immortal value if (_Py_IsImmortal(op)) { _Py_INCREF_IMMORTAL_STAT_INC(); return; diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index dee00715b3c51d..2ce3a008b7129e 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -966,6 +966,10 @@ PyAPI_FUNC(int) PyUnicode_EqualToUTF8(PyObject *, const char *); PyAPI_FUNC(int) PyUnicode_EqualToUTF8AndSize(PyObject *, const char *, Py_ssize_t); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(int) PyUnicode_Equal(PyObject *str1, PyObject *str2); +#endif + /* Rich compare two strings and return one of the following: - NULL in case an exception was raised diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 95181a420f1dfb..0a6ecf899458ed 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -11,6 +11,10 @@ The core dev team attempts to keep this documentation up to date. If it is not, please report that through the [issue tracker](https://github.com/python/cpython/issues). +Index: +----- + +[Guide to the parser](parser.md) [Compiler Design](compiler.md) @@ -20,4 +24,6 @@ it is not, please report that through the [The Source Code Locations Table](locations.md) +[Garbage collector design](garbage_collector.md) + [Exception Handling](exception_handling.md) diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index ba31e16c3bbeaa..f27e73b274511f 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -324,14 +324,14 @@ basic block. As an example, consider the following code snippet: -.. code-block:: Python - - if x < 10: - f1() - f2() - else: - g() - end() +```python +if x < 10: + f1() + f2() +else: + g() +end() +``` The ``x < 10`` guard is represented by its own basic block that compares ``x`` with ``10`` and then ends in a conditional jump based on diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index ec09e0769929fa..64a346b55b8413 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -75,7 +75,8 @@ table. If it finds a handler, control flow transfers to it. Otherwise, the exception bubbles up to the caller, and the caller's frame is checked for a handler covering the `CALL` instruction. This repeats until a handler is found or the topmost frame is reached. -If no handler is found, the program terminates. During unwinding, +If no handler is found, then the interpreter function +(``_PyEval_EvalFrameDefault()``) returns NULL. During unwinding, the traceback is constructed as each frame is added to it by ``PyTraceBack_Here()``, which is in [Python/traceback.c](https://github.com/python/cpython/blob/main/Python/traceback.c). @@ -182,3 +183,13 @@ The interpreter's function to lookup the table by instruction offset is The Python function ``_parse_exception_table()`` in [Lib/dis.py](https://github.com/python/cpython/blob/main/Lib/dis.py) returns the exception table content as a list of namedtuple instances. + +Exception Chaining Implementation +--------------------------------- + +[Exception chaining](https://docs.python.org/dev/tutorial/errors.html#exception-chaining) +refers to setting the ``__context__`` and ``__cause__`` fields of an exception as it is +being raised. The ``__context__`` field is set by ``_PyErr_SetObject()`` in +[Python/errors.c](https://github.com/python/cpython/blob/main/Python/errors.c) +(which is ultimately called by all ``PyErr_Set*()`` functions). +The ``__cause__`` field (explicit chaining) is set by the ``RAISE_VARARGS`` bytecode. diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md new file mode 100644 index 00000000000000..fd0246fa1a60e2 --- /dev/null +++ b/InternalDocs/garbage_collector.md @@ -0,0 +1,596 @@ + +Garbage collector design +======================== + +Abstract +======== + +The main garbage collection algorithm used by CPython is reference counting. The basic idea is +that CPython counts how many different places there are that have a reference to an +object. Such a place could be another object, or a global (or static) C variable, or +a local variable in some C function. When an object’s reference count becomes zero, +the object is deallocated. If it contains references to other objects, their +reference counts are decremented. Those other objects may be deallocated in turn, if +this decrement makes their reference count become zero, and so on. The reference +count field can be examined using the ``sys.getrefcount()`` function (notice that the +value returned by this function is always 1 more as the function also has a reference +to the object when called): + +```pycon + >>> x = object() + >>> sys.getrefcount(x) + 2 + >>> y = x + >>> sys.getrefcount(x) + 3 + >>> del y + >>> sys.getrefcount(x) + 2 +``` + +The main problem with the reference counting scheme is that it does not handle reference +cycles. For instance, consider this code: + +```pycon + >>> container = [] + >>> container.append(container) + >>> sys.getrefcount(container) + 3 + >>> del container +``` + +In this example, ``container`` holds a reference to itself, so even when we remove +our reference to it (the variable "container") the reference count never falls to 0 +because it still has its own internal reference. Therefore it would never be +cleaned just by simple reference counting. For this reason some additional machinery +is needed to clean these reference cycles between objects once they become +unreachable. This is the cyclic garbage collector, usually called just Garbage +Collector (GC), even though reference counting is also a form of garbage collection. + +Starting in version 3.13, CPython contains two GC implementations: + +- The default build implementation relies on the + [global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) + for thread safety. +- The free-threaded build implementation pauses other executing threads when + performing a collection for thread safety. + +Both implementations use the same basic algorithms, but operate on different +data structures. The the section on +[Differences between GC implementations](#Differences-between-GC-implementations) +for the details. + + +Memory layout and object structure +================================== + +The garbage collector requires additional fields in Python objects to support +garbage collection. These extra fields are different in the default and the +free-threaded builds. + + +GC for the default build +------------------------ + +Normally the C structure supporting a regular Python object looks as follows: + +``` + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | ob_refcnt | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + +In order to support the garbage collector, the memory layout of objects is altered +to accommodate extra information **before** the normal layout: + +``` + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | *_gc_next | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyGC_Head + | *_gc_prev | | + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ob_refcnt | \ + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + + +In this way the object can be treated as a normal python object and when the extra +information associated to the GC is needed the previous fields can be accessed by a +simple type cast from the original object: `((PyGC_Head *)(the_object)-1)`. + +As is explained later in the +[Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) +section, these two extra fields are normally used to keep doubly linked lists of all the +objects tracked by the garbage collector (these lists are the GC generations, more on +that in the [Optimization: generations](#Optimization-generations) section), but +they are also reused to fulfill other purposes when the full doubly linked list +structure is not needed as a memory optimization. + +Doubly linked lists are used because they efficiently support the most frequently required operations. In +general, the collection of all objects tracked by GC is partitioned into disjoint sets, each in its own +doubly linked list. Between collections, objects are partitioned into "generations", reflecting how +often they've survived collection attempts. During collections, the generation(s) being collected +are further partitioned into, for example, sets of reachable and unreachable objects. Doubly linked lists +support moving an object from one partition to another, adding a new object, removing an object +entirely (objects tracked by GC are most often reclaimed by the refcounting system when GC +isn't running at all!), and merging partitions, all with a small constant number of pointer updates. +With care, they also support iterating over a partition while objects are being added to - and +removed from - it, which is frequently required while GC is running. + +GC for the free-threaded build +------------------------------ + +In the free-threaded build, Python objects contain a 1-byte field +``ob_gc_bits`` that is used to track garbage collection related state. The +field exists in all objects, including ones that do not support cyclic +garbage collection. The field is used to identify objects that are tracked +by the collector, ensure that finalizers are called only once per object, +and, during garbage collection, differentiate reachable vs. unreachable objects. + +``` + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | ob_tid | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | + | pad | ob_mutex | ob_gc_bits | ob_ref_local | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | ob_ref_shared | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + +Note that not all fields are to scale. ``pad`` is two bytes, ``ob_mutex`` and +``ob_gc_bits`` are each one byte, and ``ob_ref_local`` is four bytes. The +other fields, ``ob_tid``, ``ob_ref_shared``, and ``ob_type``, are all +pointer-sized (that is, eight bytes on a 64-bit platform). + + +The garbage collector also temporarily repurposes the ``ob_tid`` (thread ID) +and ``ob_ref_local`` (local reference count) fields for other purposes during +collections. + + +C APIs +------ + +Specific APIs are offered to allocate, deallocate, initialize, track, and untrack +objects with GC support. These APIs can be found in the +[Garbage Collector C API documentation](https://docs.python.org/3/c-api/gcsupport.html). + +Apart from this object structure, the type object for objects supporting garbage +collection must include the ``Py_TPFLAGS_HAVE_GC`` in its ``tp_flags`` slot and +provide an implementation of the ``tp_traverse`` handler. Unless it can be proven +that the objects cannot form reference cycles with only objects of its type or unless +the type is immutable, a ``tp_clear`` implementation must also be provided. + + +Identifying reference cycles +============================ + +The algorithm that CPython uses to detect those reference cycles is +implemented in the ``gc`` module. The garbage collector **only focuses** +on cleaning container objects (that is, objects that can contain a reference +to one or more objects). These can be arrays, dictionaries, lists, custom +class instances, classes in extension modules, etc. One could think that +cycles are uncommon but the truth is that many internal references needed by +the interpreter create cycles everywhere. Some notable examples: + +- Exceptions contain traceback objects that contain a list of frames that + contain the exception itself. +- Module-level functions reference the module's dict (which is needed to resolve globals), + which in turn contains entries for the module-level functions. +- Instances have references to their class which itself references its module, and the module + contains references to everything that is inside (and maybe other modules) + and this can lead back to the original instance. +- When representing data structures like graphs, it is very typical for them to + have internal links to themselves. + +To correctly dispose of these objects once they become unreachable, they need +to be identified first. To understand how the algorithm works, let’s take +the case of a circular linked list which has one link referenced by a +variable ``A``, and one self-referencing object which is completely +unreachable: + +```pycon + >>> import gc + + >>> class Link: + ... def __init__(self, next_link=None): + ... self.next_link = next_link + + >>> link_3 = Link() + >>> link_2 = Link(link_3) + >>> link_1 = Link(link_2) + >>> link_3.next_link = link_1 + >>> A = link_1 + >>> del link_1, link_2, link_3 + + >>> link_4 = Link() + >>> link_4.next_link = link_4 + >>> del link_4 + + # Collect the unreachable Link object (and its .__dict__ dict). + >>> gc.collect() + 2 +``` + +The GC starts with a set of candidate objects it wants to scan. In the +default build, these "objects to scan" might be all container objects or a +smaller subset (or "generation"). In the free-threaded build, the collector +always scans all container objects. + +The objective is to identify all the unreachable objects. The collector does +this by identifying reachable objects; the remaining objects must be +unreachable. The first step is to identify all of the "to scan" objects that +are **directly** reachable from outside the set of candidate objects. These +objects have a refcount larger than the number of incoming references from +within the candidate set. + +Every object that supports garbage collection will have an extra reference +count field initialized to the reference count (``gc_ref`` in the figures) +of that object when the algorithm starts. This is because the algorithm needs +to modify the reference count to do the computations and in this way the +interpreter will not modify the real reference count field. + +![gc-image1](images/python-cyclic-gc-1-new-page.png) + +The GC then iterates over all containers in the first list and decrements by one the +`gc_ref` field of any other object that container is referencing. Doing +this makes use of the ``tp_traverse`` slot in the container class (implemented +using the C API or inherited by a superclass) to know what objects are referenced by +each container. After all the objects have been scanned, only the objects that have +references from outside the “objects to scan” list will have ``gc_ref > 0``. + +![gc-image2](images/python-cyclic-gc-2-new-page.png) + +Notice that having ``gc_ref == 0`` does not imply that the object is unreachable. +This is because another object that is reachable from the outside (``gc_ref > 0``) +can still have references to it. For instance, the ``link_2`` object in our example +ended having ``gc_ref == 0`` but is referenced still by the ``link_1`` object that +is reachable from the outside. To obtain the set of objects that are really +unreachable, the garbage collector re-scans the container objects using the +``tp_traverse`` slot; this time with a different traverse function that marks objects with +``gc_ref == 0`` as "tentatively unreachable" and then moves them to the +tentatively unreachable list. The following image depicts the state of the lists in a +moment when the GC processed the ``link_3`` and ``link_4`` objects but has not +processed ``link_1`` and ``link_2`` yet. + +![gc-image3](images/python-cyclic-gc-3-new-page.png) + +Then the GC scans the next ``link_1`` object. Because it has ``gc_ref == 1``, +the gc does not do anything special because it knows it has to be reachable (and is +already in what will become the reachable list): + +![gc-image4](images/python-cyclic-gc-4-new-page.png) + +When the GC encounters an object which is reachable (``gc_ref > 0``), it traverses +its references using the ``tp_traverse`` slot to find all the objects that are +reachable from it, moving them to the end of the list of reachable objects (where +they started originally) and setting its ``gc_ref`` field to 1. This is what happens +to ``link_2`` and ``link_3`` below as they are reachable from ``link_1``. From the +state in the previous image and after examining the objects referred to by ``link_1`` +the GC knows that ``link_3`` is reachable after all, so it is moved back to the +original list and its ``gc_ref`` field is set to 1 so that if the GC visits it again, +it will know that it's reachable. To avoid visiting an object twice, the GC marks all +objects that have already been visited once (by unsetting the ``PREV_MASK_COLLECTING`` +flag) so that if an object that has already been processed is referenced by some other +object, the GC does not process it twice. + +![gc-image5](images/python-cyclic-gc-5-new-page.png) + +Notice that an object that was marked as "tentatively unreachable" and was later +moved back to the reachable list will be visited again by the garbage collector +as now all the references that that object has need to be processed as well. This +process is really a breadth first search over the object graph. Once all the objects +are scanned, the GC knows that all container objects in the tentatively unreachable +list are really unreachable and can thus be garbage collected. + +Pragmatically, it's important to note that no recursion is required by any of this, +and neither does it in any other way require additional memory proportional to the +number of objects, number of pointers, or the lengths of pointer chains. Apart from +``O(1)`` storage for internal C needs, the objects themselves contain all the storage +the GC algorithms require. + +Why moving unreachable objects is better +---------------------------------------- + +It sounds logical to move the unreachable objects under the premise that most objects +are usually reachable, until you think about it: the reason it pays isn't actually +obvious. + +Suppose we create objects A, B, C in that order. They appear in the young generation +in the same order. If B points to A, and C to B, and C is reachable from outside, +then the adjusted refcounts after the first step of the algorithm runs will be 0, 0, +and 1 respectively because the only reachable object from the outside is C. + +When the next step of the algorithm finds A, A is moved to the unreachable list. The +same for B when it's first encountered. Then C is traversed, B is moved *back* to +the reachable list. B is eventually traversed, and then A is moved back to the reachable +list. + +So instead of not moving at all, the reachable objects B and A are each moved twice. +Why is this a win? A straightforward algorithm to move the reachable objects instead +would move A, B, and C once each. The key is that this dance leaves the objects in +order C, B, A - it's reversed from the original order. On all *subsequent* scans, +none of them will move. Since most objects aren't in cycles, this can save an +unbounded number of moves across an unbounded number of later collections. The only +time the cost can be higher is the first time the chain is scanned. + +Destroying unreachable objects +============================== + +Once the GC knows the list of unreachable objects, a very delicate process starts +with the objective of completely destroying these objects. Roughly, the process +follows these steps in order: + +1. Handle and clear weak references (if any). Weak references to unreachable objects + are set to ``None``. If the weak reference has an associated callback, the callback + is enqueued to be called once the clearing of weak references is finished. We only + invoke callbacks for weak references that are themselves reachable. If both the weak + reference and the pointed-to object are unreachable we do not execute the callback. + This is partly for historical reasons: the callback could resurrect an unreachable + object and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and the weakref + are going away, so it's legitimate to say the weak reference is going away first. +2. If an object has legacy finalizers (``tp_del`` slot) move it to the + ``gc.garbage`` list. +3. Call the finalizers (``tp_finalize`` slot) and mark the objects as already + finalized to avoid calling finalizers twice if the objects are resurrected or + if other finalizers have removed the object first. +4. Deal with resurrected objects. If some objects have been resurrected, the GC + finds the new subset of objects that are still unreachable by running the cycle + detection algorithm again and continues with them. +5. Call the ``tp_clear`` slot of every object so all internal links are broken and + the reference counts fall to 0, triggering the destruction of all unreachable + objects. + +Optimization: generations +========================= + +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. The main idea behind this concept is the assumption that most +objects have a very short lifespan and can thus be collected soon after their +creation. This has proven to be very close to the reality of many Python +programs as many temporary objects are created and destroyed very quickly. + +To take advantage of this fact, all container objects are segregated into +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. + +The GC implementation for the free-threaded build does not use multiple +generations. Every collection operates on the entire heap. + +In order to decide when to run, the collector keeps track of the number of object +allocations and deallocations since the last collection. When the number of +allocations minus the number of deallocations exceeds ``threshold_0``, +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than ``threshold_1`` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the +[`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) +function: + +```pycon + >>> import gc + >>> gc.get_threshold() + (700, 10, 10) +``` + +The content of these generations can be examined using the +``gc.get_objects(generation=NUM)`` function and collections can be triggered +specifically in a generation by calling ``gc.collect(generation=NUM)``. + +```pycon + >>> import gc + >>> class MyObj: + ... pass + ... + + # Move everything to the last generation so it's easier to inspect + # the younger generations. + + >>> gc.collect() + 0 + + # Create a reference cycle. + + >>> x = MyObj() + >>> x.self = x + + # Initially the object is in the youngest generation. + + >>> gc.get_objects(generation=0) + [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] + + # After a collection of the youngest generation the object + # moves to the next generation. + + >>> gc.collect(generation=0) + 0 + >>> gc.get_objects(generation=0) + [] + >>> gc.get_objects(generation=1) + [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] +``` + +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio ``long_lived_pending / long_lived_total`` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + +Optimization: reusing fields to save memory +=========================================== + +In order to save memory, the two linked list pointers in every object with GC +support are reused for several purposes. This is a common optimization known +as "fat pointers" or "tagged pointers": pointers that carry additional data, +"folded" into the pointer, meaning stored inline in the data representing the +address, taking advantage of certain properties of memory addressing. This is +possible as most architectures align certain types of data +to the size of the data, often a word or multiple thereof. This discrepancy +leaves a few of the least significant bits of the pointer unused, which can be +used for tags or to keep other information – most often as a bit field (each +bit a separate tag) – as long as code that uses the pointer masks out these +bits before accessing memory. For example, on a 32-bit architecture (for both +addresses and word size), a word is 32 bits = 4 bytes, so word-aligned +addresses are always a multiple of 4, hence end in ``00``, leaving the last 2 bits +available; while on a 64-bit architecture, a word is 64 bits = 8 bytes, so +word-aligned addresses end in ``000``, leaving the last 3 bits available. + +The CPython GC makes use of two fat pointers that correspond to the extra fields +of ``PyGC_Head`` discussed in the `Memory layout and object structure`_ section: + +> [!WARNING] +> Because the presence of extra information, "tagged" or "fat" pointers cannot be +> dereferenced directly and the extra information must be stripped off before +> obtaining the real memory address. Special care needs to be taken with +> functions that directly manipulate the linked lists, as these functions +> normally assume the pointers inside the lists are in a consistent state. + + +- The ``_gc_prev`` field is normally used as the "previous" pointer to maintain the + doubly linked list but its lowest two bits are used to keep the flags + ``PREV_MASK_COLLECTING`` and ``_PyGC_PREV_MASK_FINALIZED``. Between collections, + the only flag that can be present is ``_PyGC_PREV_MASK_FINALIZED`` that indicates + if an object has been already finalized. During collections ``_gc_prev`` is + temporarily used for storing a copy of the reference count (``gc_ref``), in + addition to two flags, and the GC linked list becomes a singly linked list until + ``_gc_prev`` is restored. + +- The ``_gc_next`` field is used as the "next" pointer to maintain the doubly linked + list but during collection its lowest bit is used to keep the + ``NEXT_MASK_UNREACHABLE`` flag that indicates if an object is tentatively + unreachable during the cycle detection algorithm. This is a drawback to using only + doubly linked lists to implement partitions: while most needed operations are + constant-time, there is no efficient way to determine which partition an object is + currently in. Instead, when that's needed, ad hoc tricks (like the + ``NEXT_MASK_UNREACHABLE`` flag) are employed. + +Optimization: delay tracking containers +======================================= + +Certain types of containers cannot participate in a reference cycle, and so do +not need to be tracked by the garbage collector. Untracking these objects +reduces the cost of garbage collection. However, determining which objects may +be untracked is not free, and the costs must be weighed against the benefits +for garbage collection. There are two possible strategies for when to untrack +a container: + +1. When the container is created. +2. When the container is examined by the garbage collector. + +As a general rule, instances of atomic types aren't tracked and instances of +non-atomic types (containers, user-defined objects...) are. However, some +type-specific optimizations can be present in order to suppress the garbage +collector footprint of simple instances. Some examples of native types that +benefit from delayed tracking: + +- Tuples containing only immutable objects (integers, strings etc, + and recursively, tuples of immutable objects) do not need to be tracked. The + interpreter creates a large number of tuples, many of which will not survive + until garbage collection. It is therefore not worthwhile to untrack eligible + tuples at creation time. Instead, all tuples except the empty tuple are tracked + when created. During garbage collection it is determined whether any surviving + tuples can be untracked. A tuple can be untracked if all of its contents are + already not tracked. Tuples are examined for untracking in all garbage collection + cycles. It may take more than one cycle to untrack a tuple. + +- Dictionaries containing only immutable objects also do not need to be tracked. + Dictionaries are untracked when created. If a tracked item is inserted into a + dictionary (either as a key or value), the dictionary becomes tracked. During a + full garbage collection (all generations), the collector will untrack any dictionaries + whose contents are not tracked. + +The garbage collector module provides the Python function ``is_tracked(obj)``, which returns +the current tracking status of the object. Subsequent garbage collections may change the +tracking status of the object. + +```pycon + >>> gc.is_tracked(0) + False + >>> gc.is_tracked("a") + False + >>> gc.is_tracked([]) + True + >>> gc.is_tracked({}) + False + >>> gc.is_tracked({"a": 1}) + False + >>> gc.is_tracked({"a": []}) + True +``` + +Differences between GC implementations +====================================== + +This section summarizes the differences between the GC implementation in the +default build and the implementation in the free-threaded build. + +The default build implementation makes extensive use of the ``PyGC_Head`` data +structure, while the free-threaded build implementation does not use that +data structure. + +- The default build implementation stores all tracked objects in a doubly + linked list using ``PyGC_Head``. The free-threaded build implementation + instead relies on the embedded mimalloc memory allocator to scan the heap + for tracked objects. +- The default build implementation uses ``PyGC_Head`` for the unreachable + object list. The free-threaded build implementation repurposes the + ``ob_tid`` field to store a unreachable objects linked list. +- The default build implementation stores flags in the ``_gc_prev`` field of + ``PyGC_Head``. The free-threaded build implementation stores these flags + in ``ob_gc_bits``. + + +The default build implementation relies on the +[global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) +for thread safety. The free-threaded build implementation has two "stop the +world" pauses, in which all other executing threads are temporarily paused so +that the GC can safely access reference counts and object attributes. + +The default build implementation is a generational collector. The +free-threaded build is non-generational; each collection scans the entire +heap. + +- Keeping track of object generations is simple and inexpensive in the default + build. The free-threaded build relies on mimalloc for finding tracked + objects; identifying "young" objects without scanning the entire heap would + be more difficult. + + +> [!NOTE] +> **Document history** +> +> Pablo Galindo Salgado - Original author +> +> Irit Katriel - Convert to Markdown diff --git a/InternalDocs/images/python-cyclic-gc-1-new-page.png b/InternalDocs/images/python-cyclic-gc-1-new-page.png new file mode 100644 index 00000000000000..2ddac50f4b5575 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-1-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-2-new-page.png b/InternalDocs/images/python-cyclic-gc-2-new-page.png new file mode 100644 index 00000000000000..159aeeb05024a3 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-2-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-3-new-page.png b/InternalDocs/images/python-cyclic-gc-3-new-page.png new file mode 100644 index 00000000000000..29fab0498e5b10 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-3-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-4-new-page.png b/InternalDocs/images/python-cyclic-gc-4-new-page.png new file mode 100644 index 00000000000000..51a2b1065ea64e Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-4-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-5-new-page.png b/InternalDocs/images/python-cyclic-gc-5-new-page.png new file mode 100644 index 00000000000000..fe67a6896fe4b0 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-5-new-page.png differ diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md new file mode 100644 index 00000000000000..11aaf11253646d --- /dev/null +++ b/InternalDocs/parser.md @@ -0,0 +1,894 @@ + +Guide to the parser +=================== + +Abstract +-------- + +Python's Parser is currently a +[`PEG` (Parser Expression Grammar)](https://en.wikipedia.org/wiki/Parsing_expression_grammar) +parser. It was introduced in +[PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/) to replace +the original [``LL(1)``](https://en.wikipedia.org/wiki/LL_parser) parser. + +The code implementing the parser is generated from a grammar definition by a +[parser generator](https://en.wikipedia.org/wiki/Compiler-compiler). +Therefore, changes to the Python language are made by modifying the +[grammar file](https://github.com/python/cpython/blob/main/Grammar/python.gram). +Developers rarely need to modify the generator itself. + +See the devguide's [Changing CPython's grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) +for a detailed description of the grammar and the process for changing it. + +How PEG parsers work +==================== + +A PEG (Parsing Expression Grammar) grammar differs from a +[context-free grammar](https://en.wikipedia.org/wiki/Context-free_grammar) +in that the way it is written more closely reflects how the parser will operate +when parsing. The fundamental technical difference is that the choice operator +is ordered. This means that when writing: + +``` + rule: A | B | C +``` + +a parser that implements a context-free-grammar (such as an ``LL(1)`` parser) will +generate constructions that, given an input string, *deduce* which alternative +(``A``, ``B`` or ``C``) must be expanded. On the other hand, a PEG parser will +check each alternative, in the order in which they are specified, and select +that first one that succeeds. + +This means that in a PEG grammar, the choice operator is not commutative. +Furthermore, unlike context-free grammars, the derivation according to a +PEG grammar cannot be ambiguous: if a string parses, it has exactly one +valid parse tree. + +PEG parsers are usually constructed as a recursive descent parser in which every +rule in the grammar corresponds to a function in the program implementing the +parser, and the parsing expression (the "expansion" or "definition" of the rule) +represents the "code" in said function. Each parsing function conceptually takes +an input string as its argument, and yields one of the following results: + +* A "success" result. This result indicates that the expression can be parsed by + that rule and the function may optionally move forward or consume one or more + characters of the input string supplied to it. +* A "failure" result, in which case no input is consumed. + +Note that "failure" results do not imply that the program is incorrect, nor do +they necessarily mean that the parsing has failed. Since the choice operator is +ordered, a failure very often merely indicates "try the following option". A +direct implementation of a PEG parser as a recursive descent parser will present +exponential time performance in the worst case, because PEG parsers have +infinite lookahead (this means that they can consider an arbitrary number of +tokens before deciding for a rule). Usually, PEG parsers avoid this exponential +time complexity with a technique called +["packrat parsing"](https://pdos.csail.mit.edu/~baford/packrat/thesis/) +which not only loads the entire program in memory before parsing it but also +allows the parser to backtrack arbitrarily. This is made efficient by memoizing +the rules already matched for each position. The cost of the memoization cache +is that the parser will naturally use more memory than a simple ``LL(1)`` parser, +which normally are table-based. + + +Key ideas +--------- + +- Alternatives are ordered ( ``A | B`` is not the same as ``B | A`` ). +- If a rule returns a failure, it doesn't mean that the parsing has failed, + it just means "try something else". +- By default PEG parsers run in exponential time, which can be optimized to linear by + using memoization. +- If parsing fails completely (no rule succeeds in parsing all the input text), the + PEG parser doesn't have a concept of "where the + [``SyntaxError``](https://docs.python.org/3/library/exceptions.html#SyntaxError) is". + + +> [!IMPORTANT] +> Don't try to reason about a PEG grammar in the same way you would to with an +> [EBNF](https://en.wikipedia.org/wiki/Extended_Backus–Naur_form) +> or context free grammar. PEG is optimized to describe **how** input strings will +> be parsed, while context-free grammars are optimized to generate strings of the +> language they describe (in EBNF, to know whether a given string is in the +> language, you need to do work to find out as it is not immediately obvious from +> the grammar). + + +Consequences of the ordered choice operator +------------------------------------------- + +Although PEG may look like EBNF, its meaning is quite different. The fact +that the alternatives are ordered in a PEG grammer (which is at the core of +how PEG parsers work) has deep consequences, other than removing ambiguity. + +If a rule has two alternatives and the first of them succeeds, the second one is +**not** attempted even if the caller rule fails to parse the rest of the input. +Thus the parser is said to be "eager". To illustrate this, consider +the following two rules (in these examples, a token is an individual character): + +``` + first_rule: ( 'a' | 'aa' ) 'a' + second_rule: ('aa' | 'a' ) 'a' +``` + +In a regular EBNF grammar, both rules specify the language ``{aa, aaa}`` but +in PEG, one of these two rules accepts the string ``aaa`` but not the string +``aa``. The other does the opposite -- it accepts the string ``aa`` +but not the string ``aaa``. The rule ``('a'|'aa')'a'`` does +not accept ``aaa`` because ``'a'|'aa'`` consumes the first ``a``, letting the +final ``a`` in the rule consume the second, and leaving out the third ``a``. +As the rule has succeeded, no attempt is ever made to go back and let +``'a'|'aa'`` try the second alternative. The expression ``('aa'|'a')'a'`` does +not accept ``aa`` because ``'aa'|'a'`` accepts all of ``aa``, leaving nothing +for the final ``a``. Again, the second alternative of ``'aa'|'a'`` is not +tried. + +> [!CAUTION] +> The effects of ordered choice, such as the ones illustrated above, may be +> hidden by many levels of rules. + +For this reason, writing rules where an alternative is contained in the next +one is in almost all cases a mistake, for example: + +``` + my_rule: + | 'if' expression 'then' block + | 'if' expression 'then' block 'else' block +``` + +In this example, the second alternative will never be tried because the first one will +succeed first (even if the input string has an ``'else' block`` that follows). To correctly +write this rule you can simply alter the order: + +``` + my_rule: + | 'if' expression 'then' block 'else' block + | 'if' expression 'then' block +``` + +In this case, if the input string doesn't have an ``'else' block``, the first alternative +will fail and the second will be attempted. + +Grammar Syntax +============== + +The grammar consists of a sequence of rules of the form: + +``` + rule_name: expression +``` + +Optionally, a type can be included right after the rule name, which +specifies the return type of the C or Python function corresponding to +the rule: + +``` + rule_name[return_type]: expression +``` + +If the return type is omitted, then a ``void *`` is returned in C and an +``Any`` in Python. + +Grammar expressions +------------------- + +| Expression | Description and Example | +|-----------------|-----------------------------------------------------------------------------------------------------------------------| +| `# comment` | Python-style comments. | +| `e1 e2` | Match `e1`, then match `e2`.
`rule_name: first_rule second_rule` | +| `e1 \| e2` | Match `e1` or `e2`.
`rule_name[return_type]:`
` \| first_alt`
` \| second_alt` | +| `( e )` | Grouping operator: Match `e`.
`rule_name: (e)`
`rule_name: (e1 e2)*` | +| `[ e ]` or `e?` | Optionally match `e`.
`rule_name: [e]`
`rule_name: e (',' e)* [',']` | +| `e*` | Match zero or more occurrences of `e`.
`rule_name: (e1 e2)*` | +| `e+` | Match one or more occurrences of `e`.
`rule_name: (e1 e2)+` | +| `s.e+` | Match one or more occurrences of `e`, separated by `s`.
`rule_name: ','.e+` | +| `&e` | Positive lookahead: Succeed if `e` can be parsed, without consuming input. | +| `!e` | Negative lookahead: Fail if `e` can be parsed, without consuming input.
`primary: atom !'.' !'(' !'['` | +| `~` | Commit to the current alternative, even if it fails to parse (cut).
`rule_name: '(' ~ some_rule ')' \| some_alt` | + + +Left recursion +-------------- + +PEG parsers normally do not support left recursion, but CPython's parser +generator implements a technique similar to the one described in +[Medeiros et al.](https://arxiv.org/pdf/1207.0443) but using the memoization +cache instead of static variables. This approach is closer to the one described +in [Warth et al.](http://web.cs.ucla.edu/~todd/research/pepm08.pdf). This +allows us to write not only simple left-recursive rules but also more +complicated rules that involve indirect left-recursion like: + +``` + rule1: rule2 | 'a' + rule2: rule3 | 'b' + rule3: rule1 | 'c' +``` + +and "hidden left-recursion" like: + +``` + rule: 'optional'? rule '@' some_other_rule +``` + +Variables in the grammar +------------------------ + +A sub-expression can be named by preceding it with an identifier and an +``=`` sign. The name can then be used in the action (see below), like this: + +``` + rule_name[return_type]: '(' a=some_other_rule ')' { a } +``` + +Grammar actions +--------------- + +To avoid the intermediate steps that obscure the relationship between the +grammar and the AST generation, the PEG parser allows directly generating AST +nodes for a rule via grammar actions. Grammar actions are language-specific +expressions that are evaluated when a grammar rule is successfully parsed. These +expressions can be written in Python or C depending on the desired output of the +parser generator. This means that if one would want to generate a parser in +Python and another in C, two grammar files should be written, each one with a +different set of actions, keeping everything else apart from said actions +identical in both files. As an example of a grammar with Python actions, the +piece of the parser generator that parses grammar files is bootstrapped from a +meta-grammar file with Python actions that generate the grammar tree as a result +of the parsing. + +In the specific case of the PEG grammar for Python, having actions allows +directly describing how the AST is composed in the grammar itself, making it +more clear and maintainable. This AST generation process is supported by the use +of some helper functions that factor out common AST object manipulations and +some other required operations that are not directly related to the grammar. + +To indicate these actions each alternative can be followed by the action code +inside curly-braces, which specifies the return value of the alternative: + +``` + rule_name[return_type]: + | first_alt1 first_alt2 { first_alt1 } + | second_alt1 second_alt2 { second_alt1 } +``` + +If the action is omitted, a default action is generated: + +- If there is a single name in the rule, it gets returned. +- If there multiple names in the rule, a collection with all parsed + expressions gets returned (the type of the collection will be different + in C and Python). + +This default behaviour is primarily made for very simple situations and for +debugging purposes. + +> [!WARNING] +> It's important that the actions don't mutate any AST nodes that are passed +> into them via variables referring to other rules. The reason for mutation +> being not allowed is that the AST nodes are cached by memoization and could +> potentially be reused in a different context, where the mutation would be +> invalid. If an action needs to change an AST node, it should instead make a +> new copy of the node and change that. + +The full meta-grammar for the grammars supported by the PEG generator is: + +``` + start[Grammar]: grammar ENDMARKER { grammar } + + grammar[Grammar]: + | metas rules { Grammar(rules, metas) } + | rules { Grammar(rules, []) } + + metas[MetaList]: + | meta metas { [meta] + metas } + | meta { [meta] } + + meta[MetaTuple]: + | "@" NAME NEWLINE { (name.string, None) } + | "@" a=NAME b=NAME NEWLINE { (a.string, b.string) } + | "@" NAME STRING NEWLINE { (name.string, literal_eval(string.string)) } + + rules[RuleList]: + | rule rules { [rule] + rules } + | rule { [rule] } + + rule[Rule]: + | rulename ":" alts NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts)) } + | rulename ":" NEWLINE INDENT more_alts DEDENT { Rule(rulename[0], rulename[1], more_alts) } + | rulename ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts) } + + rulename[RuleName]: + | NAME '[' type=NAME '*' ']' {(name.string, type.string+"*")} + | NAME '[' type=NAME ']' {(name.string, type.string)} + | NAME {(name.string, None)} + + alts[Rhs]: + | alt "|" alts { Rhs([alt] + alts.alts)} + | alt { Rhs([alt]) } + + more_alts[Rhs]: + | "|" alts NEWLINE more_alts { Rhs(alts.alts + more_alts.alts) } + | "|" alts NEWLINE { Rhs(alts.alts) } + + alt[Alt]: + | items '$' action { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=action) } + | items '$' { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=None) } + | items action { Alt(items, action=action) } + | items { Alt(items, action=None) } + + items[NamedItemList]: + | named_item items { [named_item] + items } + | named_item { [named_item] } + + named_item[NamedItem]: + | NAME '=' ~ item {NamedItem(name.string, item)} + | item {NamedItem(None, item)} + | it=lookahead {NamedItem(None, it)} + + lookahead[LookaheadOrCut]: + | '&' ~ atom {PositiveLookahead(atom)} + | '!' ~ atom {NegativeLookahead(atom)} + | '~' {Cut()} + + item[Item]: + | '[' ~ alts ']' {Opt(alts)} + | atom '?' {Opt(atom)} + | atom '*' {Repeat0(atom)} + | atom '+' {Repeat1(atom)} + | sep=atom '.' node=atom '+' {Gather(sep, node)} + | atom {atom} + + atom[Plain]: + | '(' ~ alts ')' {Group(alts)} + | NAME {NameLeaf(name.string) } + | STRING {StringLeaf(string.string)} + + # Mini-grammar for the actions + + action[str]: "{" ~ target_atoms "}" { target_atoms } + + target_atoms[str]: + | target_atom target_atoms { target_atom + " " + target_atoms } + | target_atom { target_atom } + + target_atom[str]: + | "{" ~ target_atoms "}" { "{" + target_atoms + "}" } + | NAME { name.string } + | NUMBER { number.string } + | STRING { string.string } + | "?" { "?" } + | ":" { ":" } +``` + +As an illustrative example this simple grammar file allows directly +generating a full parser that can parse simple arithmetic expressions and that +returns a valid C-based Python AST: + +``` + start[mod_ty]: a=expr_stmt* ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + expr_stmt[stmt_ty]: a=expr NEWLINE { _PyAST_Expr(a, EXTRA) } + + expr[expr_ty]: + | l=expr '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) } + | l=expr '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) } + | term + + term[expr_ty]: + | l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) } + | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) } + | factor + + factor[expr_ty]: + | '(' e=expr ')' { e } + | atom + + atom[expr_ty]: + | NAME + | NUMBER +``` + +Here ``EXTRA`` is a macro that expands to ``start_lineno, start_col_offset, +end_lineno, end_col_offset, p->arena``, those being variables automatically +injected by the parser; ``p`` points to an object that holds on to all state +for the parser. + +A similar grammar written to target Python AST objects: + +``` + start[ast.Module]: a=expr_stmt* ENDMARKER { ast.Module(body=a or [] } + expr_stmt: a=expr NEWLINE { ast.Expr(value=a, EXTRA) } + + expr: + | l=expr '+' r=term { ast.BinOp(left=l, op=ast.Add(), right=r, EXTRA) } + | l=expr '-' r=term { ast.BinOp(left=l, op=ast.Sub(), right=r, EXTRA) } + | term + + term: + | l=term '*' r=factor { ast.BinOp(left=l, op=ast.Mult(), right=r, EXTRA) } + | l=term '/' r=factor { ast.BinOp(left=l, op=ast.Div(), right=r, EXTRA) } + | factor + + factor: + | '(' e=expr ')' { e } + | atom + + atom: + | NAME + | NUMBER +``` + +Pegen +===== + +Pegen is the parser generator used in CPython to produce the final PEG parser +used by the interpreter. It is the program that can be used to read the python +grammar located in +[`Grammar/python.gram`](https://github.com/python/cpython/blob/main/Grammar/python.gram) +and produce the final C parser. It contains the following pieces: + +- A parser generator that can read a grammar file and produce a PEG parser + written in Python or C that can parse said grammar. The generator is located at + [`Tools/peg_generator/pegen`](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen). +- A PEG meta-grammar that automatically generates a Python parser which is used + for the parser generator itself (this means that there are no manually-written + parsers). The meta-grammar is located at + [`Tools/peg_generator/pegen/metagrammar.gram`](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/metagrammar.gram). +- A generated parser (using the parser generator) that can directly produce C and Python AST objects. + +The source code for Pegen lives at +[`Tools/peg_generator/pegen`](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen) +but normally all typical commands to interact with the parser generator are executed from +the main makefile. + +How to regenerate the parser +---------------------------- + +Once you have made the changes to the grammar files, to regenerate the ``C`` +parser (the one used by the interpreter) just execute: + +``` + make regen-pegen +``` + +using the ``Makefile`` in the main directory. If you are on Windows you can +use the Visual Studio project files to regenerate the parser or to execute: + +``` + ./PCbuild/build.bat --regen +``` + +The generated parser file is located at +[`Parser/parser.c`](https://github.com/python/cpython/blob/main/Parser/parser.c). + +How to regenerate the meta-parser +--------------------------------- + +The meta-grammar (the grammar that describes the grammar for the grammar files +themselves) is located at +[`Tools/peg_generator/pegen/metagrammar.gram`](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/metagrammar.gram). +Although it is very unlikely that you will ever need to modify it, if you make +any modifications to this file (in order to implement new Pegen features) you will +need to regenerate the meta-parser (the parser that parses the grammar files). +To do so just execute: + +``` + make regen-pegen-metaparser +``` + +If you are on Windows you can use the Visual Studio project files +to regenerate the parser or to execute: + +``` + ./PCbuild/build.bat --regen +``` + + +Grammatical elements and rules +------------------------------ + +Pegen has some special grammatical elements and rules: + +- Strings with single quotes (') (for example, ``'class'``) denote KEYWORDS. +- Strings with double quotes (") (for example, ``"match"``) denote SOFT KEYWORDS. +- Uppercase names (for example, ``NAME``) denote tokens in the + [`Grammar/Tokens`](https://github.com/python/cpython/blob/main/Grammar/Tokens) file. +- Rule names starting with ``invalid_`` are used for specialized syntax errors. + + - These rules are NOT used in the first pass of the parser. + - Only if the first pass fails to parse, a second pass including the invalid + rules will be executed. + - If the parser fails in the second phase with a generic syntax error, the + location of the generic failure of the first pass will be used (this avoids + reporting incorrect locations due to the invalid rules). + - The order of the alternatives involving invalid rules matter + (like any rule in PEG). + +Tokenization +------------ + +It is common among PEG parser frameworks that the parser does both the parsing +and the tokenization, but this does not happen in Pegen. The reason is that the +Python language needs a custom tokenizer to handle things like indentation +boundaries, some special keywords like ``ASYNC`` and ``AWAIT`` (for +compatibility purposes), backtracking errors (such as unclosed parenthesis), +dealing with encoding, interactive mode and much more. Some of these reasons +are also there for historical purposes, and some others are useful even today. + +The list of tokens (all uppercase names in the grammar) that you can use can +be found in thei +[`Grammar/Tokens`](https://github.com/python/cpython/blob/main/Grammar/Tokens) +file. If you change this file to add new tokens, make sure to regenerate the +files by executing: + +``` + make regen-token +``` + +If you are on Windows you can use the Visual Studio project files to regenerate +the tokens or to execute: + +``` + ./PCbuild/build.bat --regen +``` + +How tokens are generated and the rules governing this are completely up to the tokenizer +([`Parser/lexer`](https://github.com/python/cpython/blob/main/Parser/lexer) +and +[`Parser/tokenizer`](https://github.com/python/cpython/blob/main/Parser/tokenizer)); +the parser just receives tokens from it. + +Memoization +----------- + +As described previously, to avoid exponential time complexity in the parser, +memoization is used. + +The C parser used by Python is highly optimized and memoization can be expensive +both in memory and time. Although the memory cost is obvious (the parser needs +memory for storing previous results in the cache) the execution time cost comes +for continuously checking if the given rule has a cache hit or not. In many +situations, just parsing it again can be faster. Pegen **disables memoization +by default** except for rules with the special marker ``memo`` after the rule +name (and type, if present): + +``` + rule_name[typr] (memo): + ... +``` + +By selectively turning on memoization for a handful of rules, the parser becomes +faster and uses less memory. + +> [!NOTE] +> Left-recursive rules always use memoization, since the implementation of +> left-recursion depends on it. + +To determine whether a new rule needs memoization or not, benchmarking is required +(comparing execution times and memory usage of some considerably large files with +and without memoization). There is a very simple instrumentation API available +in the generated C parse code that allows to measure how much each rule uses +memoization (check the +[`Parser/pegen.c`](https://github.com/python/cpython/blob/main/Parser/pegen.c) +file for more information) but it needs to be manually activated. + +Automatic variables +------------------- + +To make writing actions easier, Pegen injects some automatic variables in the +namespace available when writing actions. In the C parser, some of these +automatic variable names are: + +- ``p``: The parser structure. +- ``EXTRA``: This is a macro that expands to + ``(_start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena)``, + which is normally used to create AST nodes as almost all constructors need these + attributes to be provided. All of the location variables are taken from the + location information of the current token. + +Hard and soft keywords +---------------------- + +> [!NOTE] +> In the grammar files, keywords are defined using **single quotes** (for example, +> ``'class'``) while soft keywords are defined using **double quotes** (for example, +> ``"match"``). + +There are two kinds of keywords allowed in pegen grammars: *hard* and *soft* +keywords. The difference between hard and soft keywords is that hard keywords +are always reserved words, even in positions where they make no sense +(for example, ``x = class + 1``), while soft keywords only get a special +meaning in context. Trying to use a hard keyword as a variable will always +fail: + +``` + >>> class = 3 + File "", line 1 + class = 3 + ^ + SyntaxError: invalid syntax + >>> foo(class=3) + File "", line 1 + foo(class=3) + ^^^^^ + SyntaxError: invalid syntax +``` + +While soft keywords don't have this limitation if used in a context other the +one where they are defined as keywords: + +``` + >>> match = 45 + >>> foo(match="Yeah!") +``` + +The ``match`` and ``case`` keywords are soft keywords, so that they are +recognized as keywords at the beginning of a match statement or case block +respectively, but are allowed to be used in other places as variable or +argument names. + +You can get a list of all keywords defined in the grammar from Python: + +``` + >>> import keyword + >>> keyword.kwlist + ['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', + 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', + 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', + 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] +``` + +as well as soft keywords: + +``` + >>> import keyword + >>> keyword.softkwlist + ['_', 'case', 'match'] +``` + +> [!CAUTION] +> Soft keywords can be a bit challenging to manage as they can be accepted in +> places you don't intend, given how the order alternatives behave in PEG +> parsers (see the +> [consequences of ordered choice](#consequences-of-the-ordered-choice-operator) +> section for some background on this). In general, try to define them in places +> where there are not many alternatives. + +Error handling +-------------- + +When a pegen-generated parser detects that an exception is raised, it will +**automatically stop parsing**, no matter what the current state of the parser +is, and it will unwind the stack and report the exception. This means that if a +[rule action](#grammar-actions) raises an exception, all parsing will +stop at that exact point. This is done to allow to correctly propagate any +exception set by calling Python's C API functions. This also includes +[``SyntaxError``](https://docs.python.org/3/library/exceptions.html#SyntaxError) +exceptions and it is the main mechanism the parser uses to report custom syntax +error messages. + +> [!NOTE] +> Tokenizer errors are normally reported by raising exceptions but some special +> tokenizer errors such as unclosed parenthesis will be reported only after the +> parser finishes without returning anything. + +How syntax errors are reported +------------------------------ + +As described previously in the [how PEG parsers work](#how-peg-parsers-work) +section, PEG parsers don't have a defined concept of where errors happened +in the grammar, because a rule failure doesn't imply a parsing failure like +in context free grammars. This means that a heuristic has to be used to report +generic errors unless something is explicitly declared as an error in the +grammar. + +To report generic syntax errors, pegen uses a common heuristic in PEG parsers: +the location of *generic* syntax errors is reported to be the furthest token that +was attempted to be matched but failed. This is only done if parsing has failed +(the parser returns ``NULL`` in C or ``None`` in Python) but no exception has +been raised. + +As the Python grammar was primordially written as an ``LL(1)`` grammar, this heuristic +has an extremely high success rate, but some PEG features, such as lookaheads, +can impact this. + +> [!CAUTION] +> Positive and negative lookaheads will try to match a token so they will affect +> the location of generic syntax errors. Use them carefully at boundaries +> between rules. + +To generate more precise syntax errors, custom rules are used. This is a common +practice also in context free grammars: the parser will try to accept some +construct that is known to be incorrect just to report a specific syntax error +for that construct. In pegen grammars, these rules start with the ``invalid_`` +prefix. This is because trying to match these rules normally has a performance +impact on parsing (and can also affect the 'correct' grammar itself in some +tricky cases, depending on the ordering of the rules) so the generated parser +acts in two phases: + +1. The first phase will try to parse the input stream without taking into + account rules that start with the ``invalid_`` prefix. If the parsing + succeeds it will return the generated AST and the second phase will be + skipped. + +2. If the first phase failed, a second parsing attempt is done including the + rules that start with an ``invalid_`` prefix. By design this attempt + **cannot succeed** and is only executed to give to the invalid rules a + chance to detect specific situations where custom, more precise, syntax + errors can be raised. This also allows to trade a bit of performance for + precision reporting errors: given that we know that the input text is + invalid, there is typically no need to be fast because execution is going + to stop anyway. + +> [!IMPORTANT] +> When defining invalid rules: +> +> - Make sure all custom invalid rules raise +> [``SyntaxError``](https://docs.python.org/3/library/exceptions.html#SyntaxError) +> exceptions (or a subclass of it). +> - Make sure **all** invalid rules start with the ``invalid_`` prefix to not +> impact performance of parsing correct Python code. +> - Make sure the parser doesn't behave differently for regular rules when you introduce invalid rules +> (see the [how PEG parsers work](#how-peg-parsers-work) section for more information). + +You can find a collection of macros to raise specialized syntax errors in the +[`Parser/pegen.h`](https://github.com/python/cpython/blob/main/Parser/pegen.h) +header file. These macros allow also to report ranges for +the custom errors, which will be highlighted in the tracebacks that will be +displayed when the error is reported. + + +> [!TIP] +> A good way to test whether an invalid rule will be triggered when you expect +> is to test if introducing a syntax error **after** valid code triggers the +> rule or not. For example: + +``` + $ 42 +``` + +should trigger the syntax error in the ``$`` character. If your rule is not correctly defined this +won't happen. As another example, suppose that you try to define a rule to match Python 2 style +``print`` statements in order to create a better error message and you define it as: + +``` + invalid_print: "print" expression +``` + +This will **seem** to work because the parser will correctly parse ``print(something)`` because it is valid +code and the second phase will never execute but if you try to parse ``print(something) $ 3`` the first pass +of the parser will fail (because of the ``$``) and in the second phase, the rule will match the +``print(something)`` as ``print`` followed by the variable ``something`` between parentheses and the error +will be reported there instead of the ``$`` character. + +Generating AST objects +---------------------- + +The output of the C parser used by CPython, which is generated from the +[grammar file](https://github.com/python/cpython/blob/main/Grammar/python.gram), +is a Python AST object (using C structures). This means that the actions in the +grammar file generate AST objects when they succeed. Constructing these objects +can be quite cumbersome (see the [AST compiler section](compiler.md#abstract-syntax-trees-ast) +for more information on how these objects are constructed and how they are used +by the compiler), so special helper functions are used. These functions are +declared in the +[`Parser/pegen.h`](https://github.com/python/cpython/blob/main/Parser/pegen.h) +header file and defined in the +[`Parser/action_helpers.c`](https://github.com/python/cpython/blob/main/Parser/action_helpers.c) +file. The helpers include functions that join AST sequences, get specific elements +from them or to perform extra processing on the generated tree. + + +> [!CAUTION] +> Actions must **never** be used to accept or reject rules. It may be tempting +> in some situations to write a very generic rule and then check the generated +> AST to decide whether it is valid or not, but this will render the +> (official grammar)[https://docs.python.org/3/reference/grammar.html] partially +> incorrect (because it does not include actions) and will make it more difficult +> for other Python implementations to adapt the grammar to their own needs. + +As a general rule, if an action spawns multiple lines or requires something more +complicated than a single expression of C code, is normally better to create a +custom helper in +[`Parser/action_helpers.c`](https://github.com/python/cpython/blob/main/Parser/action_helpers.c) +and expose it in the +[`Parser/pegen.h`](https://github.com/python/cpython/blob/main/Parser/pegen.h) +header file so that it can be used from the grammar. + +When parsing succeeds, the parser **must** return a **valid** AST object. + +Testing +======= + +There are three files that contain tests for the grammar and the parser: + +- [test_grammar.py](https://github.com/python/cpython/blob/main/Lib/test/test_grammar.py) +- [test_syntax.py](https://github.com/python/cpython/blob/main/Lib/test/test_syntax.py) +- [test_exceptions.py](https://github.com/python/cpython/blob/main/Lib/test/test_exceptions.py) + +Check the contents of these files to know which is the best place for new tests, depending +on the nature of the new feature you are adding. + +Tests for the parser generator itself can be found in the +[test_peg_generator](https://github.com/python/cpython/blob/main/Lib/test_peg_generator) +directory. + + +Debugging generated parsers +=========================== + +Making experiments +------------------ + +As the generated C parser is the one used by Python, this means that if +something goes wrong when adding some new rules to the grammar, you cannot +correctly compile and execute Python anymore. This makes it a bit challenging +to debug when something goes wrong, especially when experimenting. + +For this reason it is a good idea to experiment first by generating a Python +parser. To do this, you can go to the +[Tools/peg_generator](https://github.com/python/cpython/blob/main/Tools/peg_generator) +directory on the CPython repository and manually call the parser generator by executing: + +``` + $ python -m pegen python +``` + +This will generate a file called ``parse.py`` in the same directory that you +can use to parse some input: + +``` + $ python parse.py file_with_source_code_to_test.py +``` + +As the generated ``parse.py`` file is just Python code, you can modify it +and add breakpoints to debug or better understand some complex situations. + + +Verbose mode +------------ + +When Python is compiled in debug mode (by adding ``--with-pydebug`` when +running the configure step in Linux or by adding ``-d`` when calling the +[PCbuild/build.bat](https://github.com/python/cpython/blob/main/PCbuild/build.bat)), +it is possible to activate a **very** verbose mode in the generated parser. This +is very useful to debug the generated parser and to understand how it works, but it +can be a bit hard to understand at first. + +> [!NOTE] +> When activating verbose mode in the Python parser, it is better to not use +> interactive mode as it can be much harder to understand, because interactive +> mode involves some special steps compared to regular parsing. + +To activate verbose mode you can add the ``-d`` flag when executing Python: + +``` + $ python -d file_to_test.py +``` + +This will print **a lot** of output to ``stderr`` so it is probably better to dump +it to a file for further analysis. The output consists of trace lines with the +following structure:: + +``` + ('>'|'-'|'+'|'!') []: ... +``` + +Every line is indented by a different amount (````) depending on how +deep the call stack is. The next character marks the type of the trace: + +- ``>`` indicates that a rule is going to be attempted to be parsed. +- ``-`` indicates that a rule has failed to be parsed. +- ``+`` indicates that a rule has been parsed correctly. +- ``!`` indicates that an exception or an error has been detected and the parser is unwinding. + +The ```` part indicates the current index in the token array, +the ```` part indicates what rule is being parsed and +the ```` part indicates what alternative within that rule +is being attempted. + + +> [!NOTE] +> **Document history** +> +> Pablo Galindo Salgado - Original author +> Irit Katriel and Jacob Coffee - Convert to Markdown diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 358e2c070cd5fa..e0d20632516142 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -72,7 +72,7 @@ We currently also immortalize strings contained in code objects and similar, specifically in the compiler and in `marshal`. These are “close enough” to immortal: even in use cases like hot reloading or `eval`-ing user input, the number of distinct identifiers and string -constants expected to stay low. +constants is expected to stay low. ## Internal API diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 75252b3a87f9c4..06667b7434ccef 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -485,9 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() + from annotationlib import value_to_string return (f'collections.abc.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([value_to_string(a) for a in self.__args__[:-1]])}], ' + f'{value_to_string(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ @@ -524,23 +525,6 @@ def _is_param_expr(obj): names = ('ParamSpec', '_ConcatenateGenericAlias') return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - Copied from :mod:`typing` since collections.abc - shouldn't depend on that module. - (Keep this roughly in sync with the typing version.) - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is Ellipsis: - return '...' - if isinstance(obj, FunctionType): - return obj.__name__ - return repr(obj) - class Callable(metaclass=ABCMeta): @@ -978,7 +962,7 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 6e4b33921863cb..9a793717cf082b 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -315,7 +315,6 @@ 'UNPACK_EX': 113, 'UNPACK_SEQUENCE': 114, 'YIELD_VALUE': 115, - '_DO_CALL_FUNCTION_EX': 116, 'INSTRUMENTED_END_FOR': 236, 'INSTRUMENTED_END_SEND': 237, 'INSTRUMENTED_LOAD_SUPER_ATTR': 238, @@ -335,13 +334,15 @@ 'INSTRUMENTED_CALL': 252, 'INSTRUMENTED_JUMP_BACKWARD': 253, 'JUMP': 256, - 'JUMP_NO_INTERRUPT': 257, - 'LOAD_CLOSURE': 258, - 'POP_BLOCK': 259, - 'SETUP_CLEANUP': 260, - 'SETUP_FINALLY': 261, - 'SETUP_WITH': 262, - 'STORE_FAST_MAYBE_NULL': 263, + 'JUMP_IF_FALSE': 257, + 'JUMP_IF_TRUE': 258, + 'JUMP_NO_INTERRUPT': 259, + 'LOAD_CLOSURE': 260, + 'POP_BLOCK': 261, + 'SETUP_CLEANUP': 262, + 'SETUP_FINALLY': 263, + 'SETUP_WITH': 264, + 'STORE_FAST_MAYBE_NULL': 265, } HAVE_ARGUMENT = 41 diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index f8e121eb79a04d..78e03e32896740 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -463,6 +463,17 @@ def _parse_isoformat_time(tstr): time_comps = _parse_hh_mm_ss_ff(timestr) + hour, minute, second, microsecond = time_comps + became_next_day = False + error_from_components = False + if (hour == 24): + if all(time_comp == 0 for time_comp in time_comps[1:]): + hour = 0 + time_comps[0] = hour + became_next_day = True + else: + error_from_components = True + tzi = None if tz_pos == len_str and tstr[-1] == 'Z': tzi = timezone.utc @@ -495,7 +506,7 @@ def _parse_isoformat_time(tstr): time_comps.append(tzi) - return time_comps + return time_comps, became_next_day, error_from_components # tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar def _isoweek_to_gregorian(year, week, day): @@ -940,6 +951,7 @@ class date: fromtimestamp() today() fromordinal() + strptime() Operators: @@ -1040,6 +1052,12 @@ def fromisocalendar(cls, year, week, day): This is the inverse of the date.isocalendar() function""" return cls(*_isoweek_to_gregorian(year, week, day)) + @classmethod + def strptime(cls, date_string, format): + """Parse a date string according to the given format (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + # Conversions to string def __repr__(self): @@ -1360,6 +1378,7 @@ class time: Constructors: __new__() + strptime() Operators: @@ -1418,6 +1437,12 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold self._fold = fold return self + @classmethod + def strptime(cls, date_string, format): + """string, format -> new time parsed from a string (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_time(cls, date_string, format) + # Read-only field accessors @property def hour(self): @@ -1588,7 +1613,7 @@ def fromisoformat(cls, time_string): time_string = time_string.removeprefix('T') try: - return cls(*_parse_isoformat_time(time_string)) + return cls(*_parse_isoformat_time(time_string)[0]) except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') @@ -1902,10 +1927,27 @@ def fromisoformat(cls, date_string): if tstr: try: - time_components = _parse_isoformat_time(tstr) + time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) except ValueError: raise ValueError( f'Invalid isoformat string: {date_string!r}') from None + else: + if error_from_components: + raise ValueError("minute, second, and microsecond must be 0 when hour is 24") + + if became_next_day: + year, month, day = date_components + # Only wrap day/month when it was previously valid + if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + # Calculate midnight of the next day + day += 1 + if day > days_in_month: + day = 1 + month += 1 + if month > 12: + month = 1 + year += 1 + date_components = [year, month, day] else: time_components = [0, 0, 0, 0, None] @@ -2124,7 +2166,7 @@ def __str__(self): def strptime(cls, date_string, format): 'string, format -> new datetime parsed from a string (like time.strptime()).' import _strptime - return _strptime._strptime_datetime(cls, date_string, format) + return _strptime._strptime_datetime_datetime(cls, date_string, format) def utcoffset(self): """Return the timezone offset as timedelta positive east of UTC (negative west of diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 75df3db262470b..5b60570c6c592a 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -582,6 +582,21 @@ def __new__(cls, value="0", context=None): raise TypeError("Cannot convert %r to Decimal" % value) + @classmethod + def from_number(cls, number): + """Converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') + """ + if isinstance(number, (int, Decimal, float)): + return cls(number) + raise TypeError("Cannot convert %r to Decimal" % number) + @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 18849b309b8605..7b6d10c008d3cb 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -238,7 +238,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: @@ -1794,6 +1794,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 3e72a56807f6fb..03266c4dfc2dd8 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -174,7 +174,13 @@ def _excepthook(self, typ, value, tb): def runsource(self, source, filename="", symbol="single"): try: - tree = ast.parse(source) + tree = self.compile.compiler( + source, + filename, + "exec", + ast.PyCF_ONLY_AST, + incomplete_input=False, + ) except (SyntaxError, OverflowError, ValueError): self.showsyntaxerror(filename, source=source) return False @@ -185,7 +191,7 @@ def runsource(self, source, filename="", symbol="single"): the_symbol = symbol if stmt is last_stmt else "exec" item = wrapper([stmt]) try: - code = self.compile.compiler(item, filename, the_symbol, dont_inherit=True) + code = self.compile.compiler(item, filename, the_symbol) except SyntaxError as e: if e.args[0] == "'await' outside function": python = os.path.basename(sys.executable) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 3c79cf61d04051..342a4b58bfd0f3 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -28,6 +28,7 @@ import _sitebuiltins import linecache import functools +import os import sys import code @@ -50,7 +51,9 @@ def check() -> str: try: _get_reader() except _error as e: - return str(e) or repr(e) or "unknown error" + if term := os.environ.get("TERM", ""): + term = f"; TERM={term}" + return str(str(e) or repr(e) or "unknown error") + term return "" @@ -159,10 +162,8 @@ def maybe_run_command(statement: str) -> bool: input_n += 1 except KeyboardInterrupt: r = _get_reader() - if r.last_command and 'isearch' in r.last_command.__name__: - r.isearch_direction = '' - r.console.forgetinput() - r.pop_input_trans() + if r.input_trans is r.isearch_trans: + r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) r.dirty = True r.refresh() diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index f7a0095d795ac6..d457d2b5a338eb 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -371,15 +371,19 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self) -> INPUT_RECORD | None: + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): raise WinError(GetLastError()) - if read.value == 0: - return None - return rec def get_event(self, block: bool = True) -> Event | None: @@ -390,10 +394,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.pop() while True: - rec = self._read_input() + rec = self._read_input(block) if rec is None: - if block: - continue return None if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: @@ -464,8 +466,8 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" - while self._read_input() is not None: - pass + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -590,6 +592,14 @@ class INPUT_RECORD(Structure): ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] ReadConsoleInput.restype = BOOL + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) InHandle = GetStdHandle(STD_INPUT_HANDLE) else: @@ -602,5 +612,7 @@ def _win_only(*args, **kwargs): ScrollConsoleScreenBuffer = _win_only SetConsoleMode = _win_only ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only OutHandle = 0 InHandle = 0 diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 3f868bcab42446..5f4d2475c0169b 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -15,6 +15,7 @@ import locale import calendar from re import compile as re_compile +from re import sub as re_sub from re import IGNORECASE from re import escape as re_escape from datetime import (date as datetime_date, @@ -28,6 +29,18 @@ def _getlang(): # Figure out what the current language is set to. return locale.getlocale(locale.LC_TIME) +def _findall(haystack, needle): + # Find all positions of needle in haystack. + if not needle: + return + i = 0 + while True: + i = haystack.find(needle, i) + if i < 0: + break + yield i + i += len(needle) + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -102,7 +115,8 @@ def __calc_am_pm(self): am_pm = [] for hour in (1, 22): time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) - am_pm.append(time.strftime("%p", time_tuple).lower()) + # br_FR has AM/PM info (' ',' '). + am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm def __calc_date_time(self): @@ -114,42 +128,130 @@ def __calc_date_time(self): # values within the format string is very important; it eliminates # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) - date_time = [None, None, None] - date_time[0] = time.strftime("%c", time_tuple).lower() - date_time[1] = time.strftime("%x", time_tuple).lower() - date_time[2] = time.strftime("%X", time_tuple).lower() - replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), - (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), - (self.a_month[3], '%b'), (self.am_pm[1], '%p'), - ('1999', '%Y'), ('99', '%y'), ('22', '%H'), - ('44', '%M'), ('55', '%S'), ('76', '%j'), - ('17', '%d'), ('03', '%m'), ('3', '%m'), - # '3' needed for when no leading zero. - ('2', '%w'), ('10', '%I')] - replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone - for tz in tz_values]) - for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): - current_format = date_time[offset] - for old, new in replacement_pairs: + time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) + replacement_pairs = [ + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I'), + # Non-ASCII digits + ('\u0661\u0669\u0669\u0669', '%Y'), + ('\u0669\u0669', '%Oy'), + ('\u0662\u0662', '%OH'), + ('\u0664\u0664', '%OM'), + ('\u0665\u0665', '%OS'), + ('\u0661\u0667', '%Od'), + ('\u0660\u0663', '%Om'), + ('\u0663', '%Om'), + ('\u0662', '%Ow'), + ('\u0661\u0660', '%OI'), + ] + date_time = [] + for directive in ('%c', '%x', '%X'): + current_format = time.strftime(directive, time_tuple).lower() + current_format = current_format.replace('%', '%%') + # The month and the day of the week formats are treated specially + # because of a possible ambiguity in some locales where the full + # and abbreviated names are equal or names of different types + # are equal. See doc of __find_month_format for more details. + lst, fmt = self.__find_weekday_format(directive) + if lst: + current_format = current_format.replace(lst[2], fmt, 1) + lst, fmt = self.__find_month_format(directive) + if lst: + current_format = current_format.replace(lst[3], fmt, 1) + if self.am_pm[1]: # Must deal with possible lack of locale info # manifesting itself as the empty string (e.g., Swedish's # lack of AM/PM info) or a platform returning a tuple of empty # strings (e.g., MacOS 9 having timezone as ('','')). - if old: - current_format = current_format.replace(old, new) + current_format = current_format.replace(self.am_pm[1], '%p') + for tz_values in self.timezone: + for tz in tz_values: + if tz: + current_format = current_format.replace(tz, "%Z") + # Transform all non-ASCII digits to digits in range U+0660 to U+0669. + current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", - 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -211,11 +313,15 @@ def __init__(self, locale_time=None): 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'}) - base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) - base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + '%': '%'} + for d in 'dmyHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + mapping['W'] = mapping['U'].replace('U', 'W') + base.__init__(mapping) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) def __seqToRE(self, to_convert, directive): """Convert a list to a regex string for matching a directive. @@ -243,28 +349,25 @@ def pattern(self, format): regex syntax are escaped. """ - processed_format = '' # The sub() call escapes all characters that might be misconstrued # as regex syntax. Cannot use re.escape since we have to deal with # format directives (%m, etc.). - regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") - format = regex_chars.sub(r"\\\1", format) - whitespace_replacement = re_compile(r'\s+') - format = whitespace_replacement.sub(r'\\s+', format) + format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format) + format = re_sub(r'\s+', r'\\s+', format) + format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR year_in_format = False day_of_month_in_format = False - while '%' in format: - directive_index = format.index('%')+1 - format_char = format[directive_index] - processed_format = "%s%s%s" % (processed_format, - format[:directive_index-1], - self[format_char]) - format = format[directive_index+1:] + def repl(m): + format_char = m[1] match format_char: case 'Y' | 'y' | 'G': + nonlocal year_in_format year_in_format = True case 'd': + nonlocal day_of_month_in_format day_of_month_in_format = True + return self[format_char] + format = re_sub(r'%(O?.)', repl, format) if day_of_month_in_format and not year_in_format: import warnings warnings.warn("""\ @@ -275,7 +378,7 @@ def pattern(self, format): See https://github.com/python/cpython/issues/70647.""", DeprecationWarning, skip_file_prefixes=(os.path.dirname(__file__),)) - return "%s%s" % (processed_format, format) + return format def compile(self, format): """Return a compiled re object for the format string.""" @@ -567,18 +670,40 @@ def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): tt = _strptime(data_string, format)[0] return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) -def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a class cls instance based on the input string and the +def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"): + """Return a date instance based on the input string and the + format string.""" + tt, _, _ = _strptime(data_string, format) + args = tt[:3] + return cls(*args) + +def _parse_tz(tzname, gmtoff, gmtoff_fraction): + tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + if tzname: + return datetime_timezone(tzdelta, tzname) + else: + return datetime_timezone(tzdelta) + +def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"): + """Return a time instance based on the input string and the format string.""" tt, fraction, gmtoff_fraction = _strptime(data_string, format) tzname, gmtoff = tt[-2:] - args = tt[:6] + (fraction,) - if gmtoff is not None: - tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) - if tzname: - tz = datetime_timezone(tzdelta, tzname) - else: - tz = datetime_timezone(tzdelta) - args += (tz,) + args = tt[3:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) - return cls(*args) +def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a datetime instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 2071755d71dfc8..d1c7fcaeec9821 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,31 +8,6 @@ __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 0a67742a2b3081..d5166170c071c4 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -1,8 +1,10 @@ """Helpers for introspecting and wrapping annotations.""" import ast +import builtins import enum import functools +import keyword import sys import types @@ -13,13 +15,15 @@ "call_evaluate_function", "get_annotate_function", "get_annotations", + "annotations_to_string", + "value_to_string", ] class Format(enum.IntEnum): VALUE = 1 FORWARDREF = 2 - SOURCE = 3 + STRING = 3 _Union = None @@ -154,8 +158,19 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): globals[param_name] = param locals.pop(param_name, None) - code = self.__forward_code__ - value = eval(code, globals=globals, locals=locals) + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + value = locals[arg] + elif arg in globals: + value = globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + else: + raise NameError(arg) + else: + code = self.__forward_code__ + value = eval(code, globals=globals, locals=locals) self.__forward_evaluated__ = True self.__forward_value__ = value return value @@ -254,7 +269,9 @@ class _Stringifier: __slots__ = _SLOTS def __init__(self, node, globals=None, owner=None, is_class=False, cell=None): - assert isinstance(node, ast.AST) + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) self.__arg__ = None self.__forward_evaluated__ = False self.__forward_value__ = None @@ -267,18 +284,38 @@ def __init__(self, node, globals=None, owner=None, is_class=False, cell=None): self.__cell__ = cell self.__owner__ = owner - def __convert(self, other): + def __convert_to_ast(self, other): if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__) return other.__ast_node__ elif isinstance(other, slice): return ast.Slice( - lower=self.__convert(other.start) if other.start is not None else None, - upper=self.__convert(other.stop) if other.stop is not None else None, - step=self.__convert(other.step) if other.step is not None else None, + lower=( + self.__convert_to_ast(other.start) + if other.start is not None + else None + ), + upper=( + self.__convert_to_ast(other.stop) + if other.stop is not None + else None + ), + step=( + self.__convert_to_ast(other.step) + if other.step is not None + else None + ), ) else: return ast.Constant(value=other) + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + def __make_new(self, node): return _Stringifier( node, self.__globals__, self.__owner__, self.__forward_is_class__ @@ -292,38 +329,37 @@ def __hash__(self): def __getitem__(self, other): # Special case, to avoid stringifying references to class-scoped variables # as '__classdict__["x"]'. - if ( - isinstance(self.__ast_node__, ast.Name) - and self.__ast_node__.id == "__classdict__" - ): + if self.__ast_node__ == "__classdict__": raise KeyError if isinstance(other, tuple): - elts = [self.__convert(elt) for elt in other] + elts = [self.__convert_to_ast(elt) for elt in other] other = ast.Tuple(elts) else: - other = self.__convert(other) + other = self.__convert_to_ast(other) assert isinstance(other, ast.AST), repr(other) - return self.__make_new(ast.Subscript(self.__ast_node__, other)) + return self.__make_new(ast.Subscript(self.__get_ast(), other)) def __getattr__(self, attr): - return self.__make_new(ast.Attribute(self.__ast_node__, attr)) + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) def __call__(self, *args, **kwargs): return self.__make_new( ast.Call( - self.__ast_node__, - [self.__convert(arg) for arg in args], + self.__get_ast(), + [self.__convert_to_ast(arg) for arg in args], [ - ast.keyword(key, self.__convert(value)) + ast.keyword(key, self.__convert_to_ast(value)) for key, value in kwargs.items() ], ) ) def __iter__(self): - yield self.__make_new(ast.Starred(self.__ast_node__)) + yield self.__make_new(ast.Starred(self.__get_ast())) def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ return ast.unparse(self.__ast_node__) def __format__(self, format_spec): @@ -332,7 +368,7 @@ def __format__(self, format_spec): def _make_binop(op: ast.AST): def binop(self, other): return self.__make_new( - ast.BinOp(self.__ast_node__, op, self.__convert(other)) + ast.BinOp(self.__get_ast(), op, self.__convert_to_ast(other)) ) return binop @@ -356,7 +392,7 @@ def binop(self, other): def _make_rbinop(op: ast.AST): def rbinop(self, other): return self.__make_new( - ast.BinOp(self.__convert(other), op, self.__ast_node__) + ast.BinOp(self.__convert_to_ast(other), op, self.__get_ast()) ) return rbinop @@ -381,9 +417,9 @@ def _make_compare(op): def compare(self, other): return self.__make_new( ast.Compare( - left=self.__ast_node__, + left=self.__get_ast(), ops=[op], - comparators=[self.__convert(other)], + comparators=[self.__convert_to_ast(other)], ) ) @@ -400,7 +436,7 @@ def compare(self, other): def _make_unary_op(op): def unary_op(self): - return self.__make_new(ast.UnaryOp(op, self.__ast_node__)) + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) return unary_op @@ -422,7 +458,7 @@ def __init__(self, namespace, globals=None, owner=None, is_class=False): def __missing__(self, key): fwdref = _Stringifier( - ast.Name(id=key), + key, globals=self.globals, owner=self.owner, is_class=self.is_class, @@ -445,7 +481,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): can be called with any of the format arguments in the Format enum, but compiler-generated __annotate__ functions only support the VALUE format. This function provides additional functionality to call __annotate__ - functions with the FORWARDREF and SOURCE formats. + functions with the FORWARDREF and STRING formats. *annotate* must be an __annotate__ function, which takes a single argument and returns a dict of annotations. @@ -463,8 +499,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): return annotate(format) except NotImplementedError: pass - if format == Format.SOURCE: - # SOURCE is implemented by calling the annotate function in a special + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special # environment where every name lookup results in an instance of _Stringifier. # _Stringifier supports every dunder operation and returns a new _Stringifier. # At the end, we get a dictionary that mostly contains _Stringifier objects (or @@ -480,7 +516,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): name = freevars[i] else: name = "__cell__" - fwdref = _Stringifier(ast.Name(id=name)) + fwdref = _Stringifier(name) new_closure.append(types.CellType(fwdref)) closure = tuple(new_closure) else: @@ -500,9 +536,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): for key, val in annos.items() } elif format == Format.FORWARDREF: - # FORWARDREF is implemented similarly to SOURCE, but there are two changes, + # FORWARDREF is implemented similarly to STRING, but there are two changes, # at the beginning and the end of the process. - # First, while SOURCE uses an empty dictionary as the namespace, so that all + # First, while STRING uses an empty dictionary as the namespace, so that all # name lookups result in _Stringifier objects, FORWARDREF uses the globals # and builtins, so that defined names map to their real values. # Second, instead of returning strings, we want to return either real values @@ -532,7 +568,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): else: name = "__cell__" fwdref = _Stringifier( - ast.Name(id=name), + name, cell=cell, owner=owner, globals=annotate.__globals__, @@ -555,6 +591,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): result = func(Format.VALUE) for obj in globals.stringifiers: obj.__class__ = ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None return result elif format == Format.VALUE: # Should be impossible because __annotate__ functions must not raise @@ -639,28 +678,36 @@ def get_annotations( if eval_str and format != Format.VALUE: raise ValueError("eval_str=True is only supported with format=Format.VALUE") - # For VALUE format, we look at __annotations__ directly. - if format != Format.VALUE: - annotate = get_annotate_function(obj) - if annotate is not None: - ann = call_annotate_function(annotate, format, owner=obj) - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") - return dict(ann) - - if isinstance(obj, type): - try: - ann = _BASE_GET_ANNOTATIONS(obj) - except AttributeError: - # For static types, the descriptor raises AttributeError. - return {} - else: - ann = getattr(obj, "__annotations__", None) - if ann is None: - return {} + match format: + case Format.VALUE: + # For VALUE, we only look at __annotations__ + ann = _get_dunder_annotations(obj) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + return dict(_get_dunder_annotations(obj)) + except NameError: + pass - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + return dict(_get_dunder_annotations(obj)) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + return annotations_to_string(ann) + case _: + raise ValueError(f"Unsupported format {format!r}") if not ann: return {} @@ -725,3 +772,57 @@ def get_annotations( for key, value in ann.items() } return return_value + + +def value_to_string(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is inteded as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, type): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + if value is ...: + return "..." + if isinstance(value, (types.FunctionType, types.BuiltinFunctionType)): + return value.__name__ + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format.""" + return { + n: t if isinstance(t, str) else value_to_string(t) + for n, t in annotations.items() + } + + +def _get_and_call_annotate(obj, format): + annotate = get_annotate_function(obj) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return dict(ann) + return None + + +def _get_dunder_annotations(obj): + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return {} + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return dict(ann) diff --git a/Lib/argparse.py b/Lib/argparse.py index 694c46db61d177..fa9f5211257e96 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -527,8 +527,7 @@ def _format_action(self, action): def _format_action_invocation(self, action): if not action.option_strings: default = self._get_default_metavar_for_positional(action) - metavar, = self._metavar_formatter(action, default)(1) - return metavar + return ' '.join(self._metavar_formatter(action, default)(1)) else: @@ -548,8 +547,7 @@ def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: result = action.metavar elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) + result = '{%s}' % ','.join(map(str, action.choices)) else: result = default_metavar @@ -589,17 +587,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params + params['choices'] = ', '.join(map(str, params['choices'])) + return help_string % params def _iter_indented_subactions(self, action): try: @@ -703,11 +703,19 @@ def _get_action_name(argument): elif argument.option_strings: return '/'.join(argument.option_strings) elif argument.metavar not in (None, SUPPRESS): - return argument.metavar + metavar = argument.metavar + if not isinstance(metavar, tuple): + return metavar + if argument.nargs == ZERO_OR_MORE and len(metavar) == 2: + return '%s[, %s]' % metavar + elif argument.nargs == ONE_OR_MORE: + return '%s[, %s]' % metavar + else: + return ', '.join(metavar) elif argument.dest not in (None, SUPPRESS): return argument.dest elif argument.choices: - return '{' + ','.join(argument.choices) + '}' + return '{%s}' % ','.join(map(str, argument.choices)) else: return None @@ -1173,9 +1181,13 @@ def add_parser(self, name, *, deprecated=False, **kwargs): help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also @@ -1224,7 +1236,8 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): @@ -1359,7 +1372,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-(?:\d+(?:_\d+)*(?:\.\d+(?:_\d+)*)?|\.\d+(?:_\d+)*)$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1409,7 +1422,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise ValueError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1425,11 +1439,17 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): raise ValueError('unknown action "%s"' % (action_class,)) action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): @@ -1441,11 +1461,12 @@ def add_argument(self, *args, **kwargs): # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): + formatter = self._get_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1513,7 +1534,11 @@ def _add_container_actions(self, container): # NOTE: if add_mutually_exclusive_group ever gains title= and # description= then this code will need to be expanded as above for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( + if group._container is container: + cont = self + else: + cont = title_group_map[group._container.title] + mutex_group = cont.add_mutually_exclusive_group( required=group.required) # map the actions to their new mutex group @@ -1533,7 +1558,9 @@ def _get_positional_kwargs(self, dest, **kwargs): # mark positional arguments as required if at least one is # always required nargs = kwargs.get('nargs') - if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS, 0]: + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1623,6 +1650,14 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_formatter"): + formatter = self._get_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): @@ -1696,6 +1731,28 @@ def add_mutually_exclusive_group(self, *args, **kwargs): return super().add_mutually_exclusive_group(*args, **kwargs) +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' + + class ArgumentParser(_AttributeHolder, _ActionsContainer): """Object for parsing command line strings into Python objects. @@ -1739,11 +1796,7 @@ def __init__(self, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1804,8 +1857,8 @@ def add_subparsers(self, **kwargs): kwargs.setdefault('parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) + title = kwargs.pop('title', _('subcommands')) + description = kwargs.pop('description', None) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals @@ -1822,6 +1875,7 @@ def add_subparsers(self, **kwargs): # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -1927,11 +1981,11 @@ def _parse_known_args(self, arg_strings, namespace): # otherwise, add the arg to the arg strings # and note the index if it was an option else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: + option_tuples = self._parse_optional(arg_string) + if option_tuples is None: pattern = 'A' else: - option_string_indices[i] = option_tuple + option_string_indices[i] = option_tuples pattern = 'O' arg_string_pattern_parts.append(pattern) @@ -1966,8 +2020,16 @@ def take_action(action, argument_strings, option_string=None): def consume_optional(start_index): # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, sep, explicit_arg = option_tuple + option_tuples = option_string_indices[start_index] + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, sep, explicit_arg in option_tuples]) + args = {'option': arg_strings[start_index], 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + raise ArgumentError(None, msg % args) + + action, option_string, sep, explicit_arg = option_tuples[0] # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2253,7 +2315,7 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None, None + return [(action, arg_string, None, None)] # if it's just a single character, it was meant to be positional if len(arg_string) == 1: @@ -2263,25 +2325,14 @@ def _parse_optional(self, arg_string): option_string, sep, explicit_arg = arg_string.partition('=') if sep and option_string in self._option_string_actions: action = self._option_string_actions[option_string] - return action, option_string, sep, explicit_arg + return [(action, option_string, sep, explicit_arg)] # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, sep, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - raise ArgumentError(None, msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple + if option_tuples: + return option_tuples # if it was not found as an option, but it looks like a negative # number, it was meant to be positional @@ -2296,7 +2347,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None, None + return [(None, arg_string, None, None)] def _get_option_tuples(self, option_string): result = [] @@ -2319,7 +2370,9 @@ def _get_option_tuples(self, option_string): # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] @@ -2328,9 +2381,9 @@ def _get_option_tuples(self, option_string): action = self._option_string_actions[option_string] tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, None, None + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here @@ -2344,43 +2397,40 @@ def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs + # if this is an optional action, -- is not allowed + option = action.option_strings # the default (None) is assumed to be a single argument if nargs is None: - nargs_pattern = '(-*A-*)' + nargs_pattern = '([A])' if option else '(-*A-*)' # allow zero or one arguments elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' + nargs_pattern = '(A?)' if option else '(-*A?-*)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' + nargs_pattern = '(A*)' if option else '(-*[A-]*)' # allow one or more arguments elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' + nargs_pattern = '(A+)' if option else '(-*A[A-]*)' # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' + nargs_pattern = '([AO]*)' if option else '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' + nargs_pattern = '(A[AO]*)' if option else '(-*A[-AO]*)' # suppress action, like nargs=0 elif nargs == SUPPRESS: - nargs_pattern = '(-*-*)' + nargs_pattern = '()' if option else '(-*)' # all others should be integers else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') + nargs_pattern = '([AO]{%d})' % nargs if option else '((?:-*A){%d}-*)' % nargs # return the pattern return nargs_pattern @@ -2483,9 +2533,8 @@ def _get_values(self, action, arg_strings): value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2493,11 +2542,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2554,11 +2600,15 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + choices = action.choices + if choices is not None: + if isinstance(choices, str): + choices = iter(choices) + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods diff --git a/Lib/ast.py b/Lib/ast.py index a954d4a97d3c22..154d2c8c1f9ebb 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1743,7 +1743,7 @@ def unparse(ast_obj): def main(): import argparse - parser = argparse.ArgumentParser(prog='python -m ast') + parser = argparse.ArgumentParser() parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 5120140e061691..95c636f9e02866 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,6 +1,7 @@ import ast import asyncio import concurrent.futures +import contextvars import inspect import os import site @@ -22,6 +23,7 @@ def __init__(self, locals, loop): self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop + self.context = contextvars.copy_context() def runcode(self, code): global return_code @@ -55,12 +57,12 @@ def callback(): return try: - repl_future = self.loop.create_task(coro) + repl_future = self.loop.create_task(coro, context=self.context) futures._chain_future(repl_future, future) except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback) + loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 000647f57dd9e3..5dbe4b28d236d3 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -17,7 +17,6 @@ import collections.abc import concurrent.futures import errno -import functools import heapq import itertools import os @@ -1140,11 +1139,18 @@ async def create_connection( except OSError: continue else: # using happy eyeballs - sock, _, _ = await staggered.staggered_race( - (functools.partial(self._connect_sock, - exceptions, addrinfo, laddr_infos) - for addrinfo in infos), - happy_eyeballs_delay, loop=self) + sock = (await staggered.staggered_race( + ( + # can't use functools.partial as it keeps a reference + # to exceptions + lambda addrinfo=addrinfo: self._connect_sock( + exceptions, addrinfo, laddr_infos + ) + for addrinfo in infos + ), + happy_eyeballs_delay, + loop=self, + ))[0] # can't use sock, _, _ as it keeks a reference to exceptions if sock is None: exceptions = [exc for sub in exceptions for exc in sub] diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5f6fa2348726cf..c95fce035cd548 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -190,8 +190,7 @@ def result(self): the future is done and has an exception set, this exception is raised. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Result is not ready.') self.__log_traceback = False @@ -208,8 +207,7 @@ def exception(self): InvalidStateError. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Exception is not set.') self.__log_traceback = False diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 1b89236599aad7..0e63c34f60f4d9 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -3,6 +3,7 @@ import contextvars import enum import functools +import inspect import threading import signal from . import coroutines @@ -84,10 +85,7 @@ def get_loop(self): return self._loop def run(self, coro, *, context=None): - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError("a coroutine was expected, got {!r}".format(coro)) - + """Run code in the embedded event loop.""" if events._get_running_loop() is not None: # fail fast with short traceback raise RuntimeError( @@ -95,8 +93,19 @@ def run(self, coro, *, context=None): self._lazy_init() + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + if context is None: context = self._context + task = self._loop.create_task(coro, context=context) if (threading.current_thread() is threading.main_thread() diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index c3a7441a7b091d..0f4df8855a80b9 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -69,7 +69,11 @@ async def staggered_race(coro_fns, delay, *, loop=None): exceptions = [] running_tasks = [] - async def run_one_coro(previous_failed) -> None: + async def run_one_coro(ok_to_start, previous_failed) -> None: + # in eager tasks this waits for the calling task to append this task + # to running_tasks, in regular tasks this wait is a no-op that does + # not yield a future. See gh-124309. + await ok_to_start.wait() # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): @@ -85,8 +89,12 @@ async def run_one_coro(previous_failed) -> None: return # Start task that will run the next coroutine this_failed = locks.Event() - next_task = loop.create_task(run_one_coro(this_failed)) + next_ok_to_start = locks.Event() + next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) running_tasks.append(next_task) + # next_task has been appended to running_tasks so next_task is ok to + # start. + next_ok_to_start.set() assert len(running_tasks) == this_index + 2 # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) @@ -116,8 +124,11 @@ async def run_one_coro(previous_failed) -> None: if i != this_index: t.cancel() - first_task = loop.create_task(run_one_coro(None)) + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) running_tasks.append(first_task) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() try: # Wait for a growing list of tasks to all finish: poor man's version of # curio's TaskGroup or trio's nursery @@ -133,6 +144,7 @@ async def run_one_coro(previous_failed) -> None: raise d.exception() return winner_result, winner_index, exceptions finally: + del exceptions # Make sure no tasks are left running if we leave this function for t in running_tasks: t.cancel() diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index f2ee9648c43876..9fa772ca9d02cc 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -66,6 +66,20 @@ async def __aenter__(self): return self async def __aexit__(self, et, exc, tb): + tb = None + try: + return await self._aexit(et, exc) + finally: + # Exceptions are heavy objects that can have object + # cycles (bad for GC); let's not keep a reference to + # a bunch of them. It would be nicer to use a try/finally + # in __aexit__ directly but that introduced some diff noise + self._parent_task = None + self._errors = None + self._base_error = None + exc = None + + async def _aexit(self, et, exc): self._exiting = True if (exc is not None and @@ -122,7 +136,10 @@ async def __aexit__(self, et, exc, tb): assert not self._tasks if self._base_error is not None: - raise self._base_error + try: + raise self._base_error + finally: + exc = None if self._parent_cancel_requested: # If this flag is set we *must* call uncancel(). @@ -133,8 +150,14 @@ async def __aexit__(self, et, exc, tb): # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error is not None and not self._errors: - raise propagate_cancellation_error + try: + if propagate_cancellation_error is not None and not self._errors: + try: + raise propagate_cancellation_error + finally: + exc = None + finally: + propagate_cancellation_error = None if et is not None and not issubclass(et, exceptions.CancelledError): self._errors.append(exc) @@ -146,14 +169,14 @@ async def __aexit__(self, et, exc, tb): if self._parent_task.cancelling(): self._parent_task.uncancel() self._parent_task.cancel() - # Exceptions are heavy objects that can have object - # cycles (bad for GC); let's not keep a reference to - # a bunch of them. try: - me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors) - raise me from None + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None finally: - self._errors = None + exc = None + def create_task(self, coro, *, name=None, context=None): """Create a new task in this group and return it. diff --git a/Lib/bdb.py b/Lib/bdb.py index d7543017940d4f..9755d61dd2696c 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,7 @@ import fnmatch import sys import os +import weakref from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -36,6 +37,7 @@ def __init__(self, skip=None): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.code_linenos = weakref.WeakKeyDictionary() self._load_breaks() @@ -155,6 +157,9 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + # It's possible to trigger a StopIteration exception in + # the caller so we must set the trace function in the caller + self._set_caller_tracefunc(frame) return self.trace_dispatch try: self.frame_returning = frame @@ -273,9 +278,25 @@ def do_clear(self, arg): raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): - """Return True if there is any breakpoint for frame's filename. + """Return True if there is any breakpoint in that frame """ - return self.canonic(frame.f_code.co_filename) in self.breaks + filename = self.canonic(frame.f_code.co_filename) + if filename not in self.breaks: + return False + for lineno in self.breaks[filename]: + if self._lineno_in_frame(lineno, frame): + return True + return False + + def _lineno_in_frame(self, lineno, frame): + """Return True if the line number is in the frame's code object. + """ + code = frame.f_code + if lineno < code.co_firstlineno: + return False + if code not in self.code_linenos: + self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines()) + return lineno in self.code_linenos[code] # Derived classes should override the user_* methods # to gain control. @@ -329,9 +350,10 @@ def _set_caller_tracefunc(self, current_frame): # Issue #13183: pdb skips frames after hitting a breakpoint and running # step commands. # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. caller_frame = current_frame.f_back - if caller_frame and not caller_frame.f_trace: + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods @@ -360,7 +382,7 @@ def set_next(self, frame): def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: - self._set_stopinfo(frame, None, -1) + self._set_stopinfo(frame, frame, -1) else: self._set_stopinfo(frame.f_back, frame) diff --git a/Lib/codeop.py b/Lib/codeop.py index a0276b52d484e3..adf000ba29f88c 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -44,6 +44,7 @@ # Caveat emptor: These flags are undocumented on purpose and depending # on their effect outside the standard library is **unsupported**. PyCF_DONT_IMPLY_DEDENT = 0x200 +PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 def _maybe_compile(compiler, source, filename, symbol): @@ -109,12 +110,14 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol, **kwargs): - flags = self.flags + def __call__(self, source, filename, symbol, flags=0, **kwargs): + flags |= self.flags if kwargs.get('incomplete_input', True) is False: flags &= ~PyCF_DONT_IMPLY_DEDENT flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT codeob = compile(source, filename, symbol, flags, True) + if flags & PyCF_ONLY_AST: + return codeob # this is an ast.Module in this case for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index bff76291634604..034ba377a0dbec 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ -from _collections_abc import * -from _collections_abc import __all__ # noqa: F401 -from _collections_abc import _CallableGenericAlias # noqa: F401 +import _collections_abc +import sys +sys.modules[__name__] = _collections_abc diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index cb3a61287bfe5d..4a368f02d851c5 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -208,8 +208,10 @@ class c_longdouble(_SimpleCData): try: class c_double_complex(_SimpleCData): _type_ = "C" + _check_size(c_double_complex) class c_float_complex(_SimpleCData): _type_ = "E" + _check_size(c_float_complex) class c_longdouble_complex(_SimpleCData): _type_ = "F" except AttributeError: diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index ac7d40cf2cac2e..7a24f8a9e5ccee 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -283,11 +283,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -300,6 +301,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None @recursive_repr() @@ -315,6 +317,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -382,7 +385,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -394,7 +397,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -402,7 +405,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -690,11 +693,8 @@ def _frozen_get_del_attr(cls, fields, func_builder): def _is_classvar(a_type, typing): - # This test uses a typing internal class, but it's the best way to - # test if this is a ClassVar. return (a_type is typing.ClassVar - or (type(a_type) is typing._GenericAlias - and a_type.__origin__ is typing.ClassVar)) + or (typing.get_origin(a_type) is typing.ClassVar)) def _is_initvar(a_type, dataclasses): @@ -1177,7 +1177,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1221,9 +1221,56 @@ def _get_slots(cls): raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") -def _add_slots(cls, is_frozen, weakref_slot): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots.update({slot: doc}) + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: @@ -1236,17 +1283,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: @@ -1262,18 +1301,37 @@ def _add_slots(cls, is_frozen, weakref_slot): # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. if '__getstate__' not in cls_dict: - cls.__getstate__ = _dataclass_getstate + newcls.__getstate__ = _dataclass_getstate if '__setstate__' not in cls_dict: - cls.__setstate__ = _dataclass_setstate - - return cls + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, @@ -1492,7 +1550,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None): + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1572,8 +1630,8 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + # Apply the normal provided decorator. + return decorator(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, match_args=match_args, kw_only=kw_only, slots=slots, weakref_slot=weakref_slot) diff --git a/Lib/decimal.py b/Lib/decimal.py index f8c548eb1c6ecf..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -103,6 +103,7 @@ from _decimal import __version__ # noqa: F401 from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __version__ # noqa: F401 - from _pydecimal import __libmpdec_version__ # noqa: F401 + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/dis.py b/Lib/dis.py index f8832b30497822..e87e6a78469ab0 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -32,7 +32,7 @@ CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] -FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] LOAD_CONST = opmap['LOAD_CONST'] diff --git a/Lib/doctest.py b/Lib/doctest.py index ea7d275c91db04..bb281fc483c41c 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -389,11 +389,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index c5350df270487a..585afc85836c06 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -205,7 +205,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328d2f5..4183c28a809008 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/enum.py b/Lib/enum.py index 9d53eb86bc2116..17d72738792982 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1092,6 +1092,21 @@ def _add_member_(cls, name, member): # now add to _member_map_ (even aliases) cls._member_map_[name] = member + @property + def __signature__(cls): + from inspect import Parameter, Signature + if cls._member_names_: + return Signature([Parameter('values', Parameter.VAR_POSITIONAL)]) + else: + return Signature([Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None)]) + + EnumMeta = EnumType # keep EnumMeta name for backwards compatibility @@ -1135,13 +1150,6 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ - @classmethod - def __signature__(cls): - if cls._member_names_: - return '(*values)' - else: - return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' - def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' diff --git a/Lib/fractions.py b/Lib/fractions.py index 34fd0803d1b1ab..f0cbc8c2e6c012 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -279,7 +279,8 @@ def __new__(cls, numerator=0, denominator=None): numerator = -numerator else: - raise TypeError("argument should be a string or a number") + raise TypeError("argument should be a string or a Rational " + "instance or have the as_integer_ratio() method") elif type(numerator) is int is type(denominator): pass # *very* normal case @@ -305,6 +306,28 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = denominator return self + @classmethod + def from_number(cls, number): + """Converts a finite real number to a rational number, exactly. + + Beware that Fraction.from_number(0.3) != Fraction(3, 10). + + """ + if type(number) is int: + return cls._from_coprime_ints(number, 1) + + elif isinstance(number, numbers.Rational): + return cls._from_coprime_ints(number.numerator, number.denominator) + + elif (isinstance(number, float) or + (not isinstance(number, type) and + hasattr(number, 'as_integer_ratio'))): + return cls._from_coprime_ints(*number.as_integer_ratio()) + + else: + raise TypeError("argument should be a Rational instance or " + "have the as_integer_ratio() method") + @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. diff --git a/Lib/functools.py b/Lib/functools.py index 49ea9a2f6999f5..9d53d3601559b2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,24 +6,22 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Copyright (C) 2006-2024 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property'] + 'cached_property', 'Placeholder'] from abc import get_cache_token from collections import namedtuple -# import types, weakref # Deferred to single_dispatch() +# import weakref # Deferred to single_dispatch() +from operator import itemgetter from reprlib import recursive_repr -from types import MethodType +from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock -# Avoid importing types, so we can speedup import time -GenericAlias = type(list[int]) - ################################################################################ ### update_wrapper() and wraps() decorator ################################################################################ @@ -274,43 +272,125 @@ def reduce(function, sequence, initial=_initial_missing): ### partial() argument application ################################################################################ -# Purely functional, no descriptor behaviour -class partial: - """New function with partial application of the given arguments - and keywords. + +class _PlaceholderType: + """The type of the Placeholder singleton. + + Used as a placeholder for partial arguments. """ + __instance = None + __slots__ = () + + def __init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") + + def __new__(cls): + if cls.__instance is None: + cls.__instance = object.__new__(cls) + return cls.__instance - __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + def __repr__(self): + return 'Placeholder' - def __new__(cls, func, /, *args, **keywords): + def __reduce__(self): + return 'Placeholder' + +Placeholder = _PlaceholderType() + +def _partial_prepare_merger(args): + if not args: + return 0, None + nargs = len(args) + order = [] + j = nargs + for i, a in enumerate(args): + if a is Placeholder: + order.append(j) + j += 1 + else: + order.append(i) + phcount = j - nargs + merger = itemgetter(*order) if phcount else None + return phcount, merger + +def _partial_new(cls, func, /, *args, **keywords): + if issubclass(cls, partial): + base_cls = partial if not callable(func): raise TypeError("the first argument must be callable") + else: + base_cls = partialmethod + # func could be a descriptor like classmethod which isn't callable + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + if isinstance(func, base_cls): + pto_phcount = func._phcount + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] + phcount, merger = _partial_prepare_merger(tot_args) + else: # works for both pto_phcount == 0 and != 0 + phcount, merger = pto_phcount, func._merger + keywords = {**func.keywords, **keywords} + func = func.func + else: + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self + +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" - if isinstance(func, partial): - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ - self = super(partial, cls).__new__(cls) + __slots__ = ("func", "args", "keywords", "_phcount", "_merger", + "__dict__", "__weakref__") - self.func = func - self.args = args - self.keywords = keywords - return self + __new__ = _partial_new + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) - - @recursive_repr() - def __repr__(self): - cls = type(self) - qualname = cls.__qualname__ - module = cls.__module__ - args = [repr(self.func)] - args.extend(repr(x) for x in self.args) - args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + return self.func(*pto_args, *args, **keywords) def __get__(self, obj, objtype=None): if obj is None: @@ -332,6 +412,10 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount, merger = _partial_prepare_merger(args) + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} @@ -344,53 +428,40 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds + self._phcount = phcount + self._merger = merger try: - from _functools import partial + from _functools import partial, Placeholder, _PlaceholderType except ImportError: pass # Descriptor version -class partialmethod(object): +class partialmethod: """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - - def __init__(self, func, /, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords - - def __repr__(self): - cls = type(self) - module = cls.__module__ - qualname = cls.__qualname__ - args = [repr(self.func)] - args.extend(map(repr, self.args)) - args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + __new__ = _partial_new + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partialmethod' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *self.args, *args, **keywords) + return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.__partialmethod__ = self return _method @@ -826,7 +897,7 @@ def singledispatch(func): # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. - import types, weakref + import weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() @@ -857,7 +928,7 @@ def dispatch(cls): def _is_union_type(cls): from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} + return get_origin(cls) in {Union, UnionType} def _is_valid_dispatch_type(cls): if isinstance(cls, type): @@ -934,7 +1005,7 @@ def wrapper(*args, **kw): registry[object] = func wrapper.register = register wrapper.dispatch = dispatch - wrapper.registry = types.MappingProxyType(registry) + wrapper.registry = MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper diff --git a/Lib/gzip.py b/Lib/gzip.py index ba753ce3050dd8..1a3c82ce7e0711 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -580,12 +580,12 @@ def _rewind(self): self._new_member = True -def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None): +def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0): """Compress data in one shot and return the compressed string. compresslevel sets the compression level in range of 0-9. - mtime can be used to set the modification time. The modification time is - set to the current time by default. + mtime can be used to set the modification time. + The modification time is set to 0 by default, for reproducibility. """ # Wbits=31 automatically includes a gzip header and trailer. gzip_data = zlib.compress(data, level=compresslevel, wbits=31) diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 827d230b54e159..2a4adc6a4d395f 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.13.0a2 documentation + IDLE — Python 3.14.0a0 documentation @@ -18,7 +18,7 @@ @@ -26,6 +26,7 @@ + @@ -45,6 +46,8 @@ + + @@ -184,7 +187,7 @@

Navigation

  • - 3.13.0a2 Documentation » + 3.14.0a0 Documentation »
  • The Python Standard Library »
    • @@ -554,7 +557,7 @@

    Key bindingsControl key on Windows and -Unix and the Command key on macOS. (And all such dicussions +Unix and the Command key on macOS. (And all such discussions assume that the keys have not been re-bound to something else.)

     @@ -694,7 +697,7 @@

    Shell window -

  • C-c attemps to interrupt statement execution (but may fail).

    • +

  • C-c attempts to interrupt statement execution (but may fail).

  • C-d closes Shell if typed at a >>> prompt.

  • Alt-p and Alt-n (C-p and C-n on macOS) retrieve to the current prompt the previous or next previously @@ -1136,7 +1139,7 @@

    Navigation

  • - 3.13.0a2 Documentation » + 3.14.0a0 Documentation »
  • The Python Standard Library »
    • @@ -1180,7 +1183,7 @@

    Navigation



    - Last updated on Jan 17, 2024 (06:57 UTC). + Last updated on Oct 14, 2024 (20:27 UTC). Found a bug?
    diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 8f98e73258e778..a30db99a619a93 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -108,11 +108,11 @@ def handle_tk_events(tcl=tcl): # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global -# completion, exit and interruptable (the main thread) flags: +# completion, exit and interruptible (the main thread) flags: exit_now = False quitting = False -interruptable = False +interruptible = False def main(del_exitfunc=False): """Start the Python execution server in a subprocess @@ -582,14 +582,14 @@ def __init__(self, rpchandler): self.locals = {} def runcode(self, code): - global interruptable + global interruptible try: self.user_exc_info = None - interruptable = True + interruptible = True try: exec(code, self.locals) finally: - interruptable = False + interruptible = False except SystemExit as e: if e.args: # SystemExit called with an argument. ob = e.args[0] @@ -615,7 +615,7 @@ def runcode(self, code): flush_stdout() def interrupt_the_server(self): - if interruptable: + if interruptible: thread.interrupt_main() def start_the_debugger(self, gui_adap_oid): diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 0726d7e23660f6..182ce7189614da 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -83,6 +83,8 @@ def wheel_event(event, widget=None): class TreeNode: + dy = 0 + def __init__(self, canvas, parent, item): self.canvas = canvas self.parent = parent @@ -199,23 +201,22 @@ def update(self): def draw(self, x, y): # XXX This hard-codes too many geometry constants! - dy = 20 self.x, self.y = x, y self.drawicon() self.drawtext() if self.state != 'expanded': - return y + dy + return y + TreeNode.dy # draw children if not self.children: sublist = self.item._GetSubList() if not sublist: # _IsExpandable() was mistaken; that's allowed - return y+17 + return y + TreeNode.dy for item in sublist: child = self.__class__(self.canvas, self, item) self.children.append(child) cx = x+20 - cy = y + dy + cy = y + TreeNode.dy cylast = 0 for child in self.children: cylast = cy @@ -289,6 +290,11 @@ def drawtext(self): self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.text_id = id + if TreeNode.dy == 0: + # The first row doesn't matter what the dy is, just measure its + # size to get the value of the subsequent dy + coords = self.canvas.bbox(id) + TreeNode.dy = max(20, coords[3] - coords[1] - 3) def select_or_edit(self, event=None): if self.selected and self.item.IsEditable(): diff --git a/Lib/inspect.py b/Lib/inspect.py index 90c44cf74007a8..0c33c6cc995a03 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -140,6 +140,7 @@ import abc +from annotationlib import Format from annotationlib import get_annotations # re-exported import ast import dis @@ -970,10 +971,12 @@ def findsource(object): if isclass(object): try: - firstlineno = vars(object)['__firstlineno__'] + lnum = vars(object)['__firstlineno__'] - 1 except (TypeError, KeyError): raise OSError('source code not available') - return lines, firstlineno - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ @@ -1317,7 +1320,9 @@ def getargvalues(frame): args, varargs, varkw = getargs(frame.f_code) return ArgInfo(args, varargs, varkw, frame.f_locals) -def formatannotation(annotation, base_module=None): +def formatannotation(annotation, base_module=None, *, quote_annotation_strings=True): + if not quote_annotation_strings and isinstance(annotation, str): + return annotation if getattr(annotation, '__module__', None) == 'typing': def repl(match): text = match.group() @@ -1930,7 +1935,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature - new_params.pop(param_name) + # However, if it is a Placeholder it is not removed + # And also looses default value + if arg_value is functools.Placeholder: + new_params[param_name] = param.replace(default=_empty) + else: + new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: @@ -1952,7 +1962,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - new_params.pop(param.name) + # Do not pop if it is a Placeholder + # also change kind to positional only + # and remove default + if arg_value is functools.Placeholder: + new_param = param.replace( + kind=_POSITIONAL_ONLY, + default=_empty + ) + new_params[param_name] = new_param + else: + new_params.pop(param_name) continue if param.kind is _KEYWORD_ONLY: @@ -2253,7 +2273,8 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): def _signature_from_function(cls, func, skip_bound_arg=True, - globals=None, locals=None, eval_str=False): + globals=None, locals=None, eval_str=False, + *, annotation_format=Format.VALUE): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2279,7 +2300,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True, positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str) + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str, + format=annotation_format) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2362,7 +2384,8 @@ def _signature_from_callable(obj, *, globals=None, locals=None, eval_str=False, - sigcls): + sigcls, + annotation_format=Format.VALUE): """Private helper function to get signature for arbitrary callable objects. @@ -2374,7 +2397,8 @@ def _signature_from_callable(obj, *, globals=globals, locals=locals, sigcls=sigcls, - eval_str=eval_str) + eval_str=eval_str, + annotation_format=annotation_format) if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) @@ -2407,18 +2431,10 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: - # since __text_signature__ is not writable on classes, __signature__ - # may contain text (or be a callable that returns text); - # if so, convert it - o_sig = sig - if not isinstance(sig, (Signature, str)) and callable(sig): - sig = sig() - if isinstance(sig, str): - sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(o_sig)) + 'attribute'.format(sig)) return sig try: @@ -2446,6 +2462,11 @@ def _signature_from_callable(obj, *, sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) + # If there were placeholders set, + # first param is transformed to positional only + if partialmethod.args.count(functools.Placeholder): + first_wrapped_param = first_wrapped_param.replace( + kind=Parameter.POSITIONAL_ONLY) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) @@ -2458,7 +2479,8 @@ def _signature_from_callable(obj, *, # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, skip_bound_arg=skip_bound_arg, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, @@ -2693,13 +2715,17 @@ def replace(self, *, name=_void, kind=_void, return type(self)(name, kind, default=default, annotation=annotation) def __str__(self): + return self._format() + + def _format(self, *, quote_annotation_strings=True): kind = self.kind formatted = self._name # Add annotation and default value if self._annotation is not _empty: - formatted = '{}: {}'.format(formatted, - formatannotation(self._annotation)) + annotation = formatannotation(self._annotation, + quote_annotation_strings=quote_annotation_strings) + formatted = '{}: {}'.format(formatted, annotation) if self._default is not _empty: if self._annotation is not _empty: @@ -2947,11 +2973,13 @@ def __init__(self, parameters=None, *, return_annotation=_empty, @classmethod def from_callable(cls, obj, *, - follow_wrapped=True, globals=None, locals=None, eval_str=False): + follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, follow_wrapper_chains=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) @property def parameters(self): @@ -3166,19 +3194,24 @@ def __repr__(self): def __str__(self): return self.format() - def format(self, *, max_width=None): + def format(self, *, max_width=None, quote_annotation_strings=True): """Create a string representation of the Signature object. If *max_width* integer is passed, signature will try to fit into the *max_width*. If signature is longer than *max_width*, all parameters will be on separate lines. + + If *quote_annotation_strings* is False, annotations + in the signature are displayed without opening and closing quotation + marks. This is useful when the signature was created with the + STRING format or when ``from __future__ import annotations`` was used. """ result = [] render_pos_only_separator = False render_kw_only_separator = True for param in self.parameters.values(): - formatted = str(param) + formatted = param._format(quote_annotation_strings=quote_annotation_strings) kind = param.kind @@ -3215,16 +3248,19 @@ def format(self, *, max_width=None): rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) + anno = formatannotation(self.return_annotation, + quote_annotation_strings=quote_annotation_strings) rendered += ' -> {}'.format(anno) return rendered -def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Get a signature object for the passed callable.""" return Signature.from_callable(obj, follow_wrapped=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) class BufferFlags(enum.IntFlag): diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 9028e517fb9f7d..1ba91384c81f27 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -9,10 +9,9 @@ def main(): - prog = 'python -m json' description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(prog=prog, description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument('infile', nargs='?', help='a JSON file to be validated or pretty-printed', default='-') diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 3781cb1aeb9ae2..6a6a7f726f7e0c 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -499,7 +499,7 @@ def as_tuple(self, value): def _is_queue_like_object(obj): """Check that *obj* implements the Queue API.""" - if isinstance(obj, queue.Queue): + if isinstance(obj, (queue.Queue, queue.SimpleQueue)): return True # defer importing multiprocessing as much as possible from multiprocessing.queues import Queue as MPQueue @@ -516,13 +516,13 @@ def _is_queue_like_object(obj): # Ideally, we would have wanted to simply use strict type checking # instead of a protocol-based type checking since the latter does # not check the method signatures. - queue_interface = [ - 'empty', 'full', 'get', 'get_nowait', - 'put', 'put_nowait', 'join', 'qsize', - 'task_done', - ] + # + # Note that only 'put_nowait' and 'get' are required by the logging + # queue handler and queue listener (see gh-124653) and that other + # methods are either optional or unused. + minimal_queue_interface = ['put_nowait', 'get'] return all(callable(getattr(obj, method, None)) - for method in queue_interface) + for method in minimal_queue_interface) class DictConfigurator(BaseConfigurator): """ diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index ddcc7e7900999e..d0a3ad00e53ad8 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -259,13 +259,12 @@ def get_start_method(self, allow_none=False): def get_all_start_methods(self): """Returns a list of the supported start methods, default first.""" - if sys.platform == 'win32': - return ['spawn'] - else: - methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] - if reduction.HAVE_SEND_HANDLE: - methods.append('forkserver') - return methods + default = self._default_context.get_start_method() + start_method_names = [default] + start_method_names.extend( + name for name in _concrete_contexts if name != default + ) + return start_method_names # @@ -320,14 +319,15 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - if sys.platform == 'darwin': - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - _default_context = DefaultContext(_concrete_contexts['spawn']) + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + _default_context = DefaultContext(_concrete_contexts['forkserver']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts['spawn']) -else: +else: # Windows class SpawnProcess(process.BaseProcess): _start_method = 'spawn' diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 1c02e4168d3a9e..a78997179820b1 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -119,9 +119,9 @@ def __init__(self, *args): paths = [] for arg in args: if isinstance(arg, PurePath): - if arg.parser is ntpath and self.parser is posixpath: + if arg.parser is not self.parser: # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + paths.append(arg.as_posix()) else: paths.extend(arg._raw_paths) else: diff --git a/Lib/pdb.py b/Lib/pdb.py index 228de489a9cef1..d9aed24bfcd8e7 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -309,7 +309,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): _last_pdb_instance = None def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, - nosigint=False, readrc=True): + nosigint=False, readrc=True, mode=None): bdb.Bdb.__init__(self, skip=skip) cmd.Cmd.__init__(self, completekey, stdin, stdout) sys.audit("pdb.Pdb") @@ -321,6 +321,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.mainpyfile = '' self._wait_for_mainpyfile = False self.tb_lineno = {} + self.mode = mode # Try to load readline if it exists try: import readline @@ -349,10 +350,6 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, pass self.commands = {} # associates a command list to breakpoint numbers - self.commands_doprompt = {} # for each bp num, tells if the prompt - # must be disp. after execing the cmd list - self.commands_silent = {} # for each bp num, tells if the stack trace - # must be disp. after execing the cmd list self.commands_defining = False # True while in the process of defining # a command list self.commands_bnum = None # The breakpoint number for which we are @@ -361,10 +358,14 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._chained_exceptions = tuple() self._chained_exception_index = 0 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): Pdb._last_pdb_instance = self if frame is None: frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + super().set_trace(frame) def sigint_handler(self, signum, frame): @@ -398,13 +399,6 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary used to be updated from the actual frame - # locals whenever the .f_locals accessor was called, so it was - # cached here to ensure that modifications were not overwritten. While - # the caching is no longer required now that f_locals is a direct proxy - # on optimized frames, it's also harmless, so the code structure has - # been left unchanged. - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) if self._chained_exceptions: @@ -439,8 +433,8 @@ def user_line(self, frame): or frame.f_lineno <= 0): return self._wait_for_mainpyfile = False - if self.bp_commands(frame): - self.interaction(frame, None) + self.bp_commands(frame) + self.interaction(frame, None) user_opcode = user_line @@ -455,18 +449,9 @@ def bp_commands(self, frame): self.currentbp in self.commands: currentbp = self.currentbp self.currentbp = 0 - lastcmd_back = self.lastcmd - self.setup(frame, None) for line in self.commands[currentbp]: - self.onecmd(line) - self.lastcmd = lastcmd_back - if not self.commands_silent[currentbp]: - self.print_stack_entry(self.stack[self.curindex]) - if self.commands_doprompt[currentbp]: - self._cmdloop() - self.forget() - return - return 1 + self.cmdqueue.append(line) + self.cmdqueue.append(f'_pdbcmd_restore_lastcmd {self.lastcmd}') def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" @@ -727,7 +712,7 @@ def _exec_in_closure(self, source, globals, locals): def default(self, line): if line[:1] == '!': line = line[1:].strip() - locals = self.curframe_locals + locals = self.curframe.f_locals globals = self.curframe.f_globals try: buffer = line @@ -865,15 +850,15 @@ def handle_command_def(self, line): cmd, arg, line = self.parseline(line) if not cmd: return False - if cmd == 'silent': - self.commands_silent[self.commands_bnum] = True - return False # continue to handle other cmd def in the cmd list - elif cmd == 'end': + if cmd == 'end': return True # end of cmd list elif cmd == 'EOF': print('') return True # end of cmd list cmdlist = self.commands[self.commands_bnum] + if cmd == 'silent': + cmdlist.append('_pdbcmd_silence_frame_status') + return False # continue to handle other cmd def in the cmd list if arg: cmdlist.append(cmd+' '+arg) else: @@ -885,7 +870,6 @@ def handle_command_def(self, line): func = self.default # one of the resuming commands if func.__name__ in self.commands_resuming: - self.commands_doprompt[self.commands_bnum] = False return True return False @@ -955,7 +939,7 @@ def _complete_expression(self, text, line, begidx, endidx): # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} if text.startswith("$"): # Complete convenience variables conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) @@ -986,7 +970,7 @@ def completedefault(self, text, line, begidx, endidx): # Use rlcompleter to do the completion state = 0 matches = [] - completer = Completer(self.curframe.f_globals | self.curframe_locals) + completer = Completer(self.curframe.f_globals | self.curframe.f_locals) while (match := completer.complete(text, state)) is not None: matches.append(match) state += 1 @@ -998,6 +982,13 @@ def _pdbcmd_print_frame_status(self, arg): self.print_stack_trace(0) self._show_display() + def _pdbcmd_silence_frame_status(self, arg): + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() + + def _pdbcmd_restore_lastcmd(self, arg): + self.lastcmd = arg + # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop @@ -1056,14 +1047,10 @@ def do_commands(self, arg): self.commands_bnum = bnum # Save old definitions for the case of a keyboard interrupt. if bnum in self.commands: - old_command_defs = (self.commands[bnum], - self.commands_doprompt[bnum], - self.commands_silent[bnum]) + old_commands = self.commands[bnum] else: - old_command_defs = None + old_commands = None self.commands[bnum] = [] - self.commands_doprompt[bnum] = True - self.commands_silent[bnum] = False prompt_back = self.prompt self.prompt = '(com) ' @@ -1072,14 +1059,10 @@ def do_commands(self, arg): self.cmdloop() except KeyboardInterrupt: # Restore old definitions. - if old_command_defs: - self.commands[bnum] = old_command_defs[0] - self.commands_doprompt[bnum] = old_command_defs[1] - self.commands_silent[bnum] = old_command_defs[2] + if old_commands: + self.commands[bnum] = old_commands else: del self.commands[bnum] - del self.commands_doprompt[bnum] - del self.commands_silent[bnum] self.error('command definition aborted, old commands restored') finally: self.commands_defining = False @@ -1148,7 +1131,7 @@ def do_break(self, arg, temporary = 0): try: func = eval(arg, self.curframe.f_globals, - self.curframe_locals) + self.curframe.f_locals) except: func = arg try: @@ -1453,7 +1436,6 @@ def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None @@ -1607,6 +1589,11 @@ def do_run(self, arg): sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run". """ + if self.mode == 'inline': + self.error('run/restart command is disabled when pdb is running in inline mode.\n' + 'Use the command line interface to enable restarting your program\n' + 'e.g. "python -m pdb myscript.py"') + return if arg: import shlex argv0 = sys.argv[0:1] @@ -1694,7 +1681,7 @@ def do_debug(self, arg): """ sys.settrace(None) globals = self.curframe.f_globals - locals = self.curframe_locals + locals = self.curframe.f_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") @@ -1739,7 +1726,7 @@ def do_args(self, arg): self._print_invalid_arg(arg) return co = self.curframe.f_code - dict = self.curframe_locals + dict = self.curframe.f_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 @@ -1759,15 +1746,15 @@ def do_retval(self, arg): if arg: self._print_invalid_arg(arg) return - if '__return__' in self.curframe_locals: - self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) + if '__return__' in self.curframe.f_locals: + self.message(self._safe_repr(self.curframe.f_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) except: self._error_exc() raise @@ -1775,7 +1762,7 @@ def _getval(self, arg): def _getval_except(self, arg, frame=None): try: if frame is None: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except BaseException as exc: @@ -2019,7 +2006,7 @@ def do_interact(self, arg): Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} console = _PdbInteractiveConsole(ns, message=self.message) console.interact(banner="*pdb interact start*", exitmsg="*exit from pdb interact command*") @@ -2091,7 +2078,7 @@ def complete_unalias(self, text, line, begidx, endidx): # List of all the commands making the program resume execution. commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return', - 'do_quit', 'do_jump'] + 'do_until', 'do_quit', 'do_jump'] # Print a traceback starting at the top stack frame. # The most recently entered frame is printed last; @@ -2350,21 +2337,22 @@ def runcall(*args, **kwds): """ return Pdb().runcall(*args, **kwds) -def set_trace(*, header=None): +def set_trace(*, header=None, commands=None): """Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console - just before debugging begins. + just before debugging begins. *commands* is an optional list of + pdb commands to run when the debugger starts. """ if Pdb._last_pdb_instance is not None: pdb = Pdb._last_pdb_instance else: - pdb = Pdb() + pdb = Pdb(mode='inline') if header is not None: pdb.message(header) - pdb.set_trace(sys._getframe().f_back) + pdb.set_trace(sys._getframe().f_back, commands=commands) # Post-Mortem interface @@ -2435,8 +2423,7 @@ def help(): def main(): import argparse - parser = argparse.ArgumentParser(prog="pdb", - usage="%(prog)s [-h] [-c command] (-m module | pyfile) [args ...]", + parser = argparse.ArgumentParser(usage="%(prog)s [-h] [-c command] (-m module | pyfile) [args ...]", description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False) @@ -2476,7 +2463,7 @@ def main(): # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. - pdb = Pdb() + pdb = Pdb(mode='cli') pdb.rcLines.extend(opts.commands) while True: try: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d376592d69d40d..c863794ea14ef9 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -71,6 +71,7 @@ class or function within a module or module in a package. If the import tokenize import urllib.parse import warnings +from annotationlib import Format from collections import deque from reprlib import Repr from traceback import format_exception_only @@ -212,12 +213,12 @@ def splitdoc(doc): def _getargspec(object): try: - signature = inspect.signature(object) + signature = inspect.signature(object, annotation_format=Format.STRING) if signature: name = getattr(object, '__name__', '') # function are always single-line and should not be formatted max_width = (80 - len(name)) if name != '' else None - return signature.format(max_width=max_width) + return signature.format(max_width=max_width, quote_annotation_strings=False) except (ValueError, TypeError): argspec = getattr(object, '__text_signature__', None) if argspec: @@ -1870,6 +1871,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1963,6 +1965,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 4643df80e44aaf..ae56c136608472 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -416,6 +416,44 @@ 'some expressions (like un-parenthesized tuple expressions) ' 'caused a\n' 'syntax error.\n', + 'assignment-expressions': 'Assignment expressions\n' + '**********************\n' + '\n' + ' assignment_expression ::= [identifier ":="] ' + 'expression\n' + '\n' + 'An assignment expression (sometimes also called a ' + '“named expression”\n' + 'or “walrus”) assigns an "expression" to an ' + '"identifier", while also\n' + 'returning the value of the "expression".\n' + '\n' + 'One common use case is when handling matched ' + 'regular expressions:\n' + '\n' + ' if matching := pattern.search(data):\n' + ' do_something(matching)\n' + '\n' + 'Or, when processing a file stream in chunks:\n' + '\n' + ' while chunk := file.read(9000):\n' + ' process(chunk)\n' + '\n' + 'Assignment expressions must be surrounded by ' + 'parentheses when used as\n' + 'expression statements and when used as ' + 'sub-expressions in slicing,\n' + 'conditional, lambda, keyword-argument, and ' + 'comprehension-if\n' + 'expressions and in "assert", "with", and ' + '"assignment" statements. In\n' + 'all other places where they can be used, ' + 'parentheses are not required,\n' + 'including in "if" and "while" statements.\n' + '\n' + 'Added in version 3.8: See **PEP 572** for more ' + 'details about\n' + 'assignment expressions.\n', 'async': 'Coroutines\n' '**********\n' '\n' diff --git a/Lib/shutil.py b/Lib/shutil.py index 89c12b76b61dfc..dd3e0e0c5da54b 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -44,11 +44,11 @@ else: _winapi = None -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 _USE_CP_SENDFILE = (hasattr(os, "sendfile") - and sys.platform.startswith(("linux", "android", "solaris"))) + and sys.platform.startswith(("linux", "android", "sunos"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 diff --git a/Lib/site.py b/Lib/site.py index cafd3ab70b2cac..07a6361fad44e5 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -491,12 +491,26 @@ def register_readline(): This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + import atexit try: import readline import rlcompleter # noqa: F401 - import _pyrepl.readline - import _pyrepl.unix_console + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False + else: + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + import _pyrepl.unix_console + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path except ImportError: return @@ -517,7 +531,6 @@ def register_readline(): pass if readline.get_current_history_length() == 0: - from _pyrepl.main import CAN_USE_PYREPL # If no history was loaded, default to .python_history, # or PYTHON_HISTORY. # The guard is necessary to avoid doubling history size at @@ -525,13 +538,17 @@ def register_readline(): # through a PYTHONSTARTUP hook, see: # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() - if os.getenv("PYTHON_BASIC_REPL") or not CAN_USE_PYREPL: - readline_module = readline - else: + + if CAN_USE_PYREPL: readline_module = _pyrepl.readline + exceptions = (OSError, *_pyrepl.unix_console._error) + else: + readline_module = readline + exceptions = OSError + try: readline_module.read_history_file(history) - except (OSError,* _pyrepl.unix_console._error): + except exceptions: pass def write_history(): diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index d9423c25e34135..cfdee61403d1fa 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -65,7 +65,6 @@ def runsource(self, source, filename="", symbol="single"): def main(*args): parser = ArgumentParser( description="Python sqlite3 CLI", - prog="python -m sqlite3", ) parser.add_argument( "filename", type=str, default=":memory:", nargs="?", diff --git a/Lib/statistics.py b/Lib/statistics.py index d3dd0d530c31cf..e2b59267f04f68 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -248,6 +248,7 @@ def count_positive(iterable): found_zero = True else: raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) if not n: @@ -710,6 +711,7 @@ def correlation(x, y, /, *, method='linear'): start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) + else: xbar = fsum(x) / n ybar = fsum(y) / n @@ -870,9 +872,12 @@ def f_inv(y): return f_inv def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: + if p < 0.499: x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) return x * sign @@ -886,8 +891,11 @@ def quartic_kernel(): return pdf, cdf, invcdf, support def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) return x * sign @register('triweight') @@ -1207,91 +1215,6 @@ def quantiles(data, *, n=4, method='exclusive'): ## Normal Distribution ##################################################### -def _normal_dist_inv_cdf(p, mu, sigma): - # There is no closed-form solution to the inverse CDF for the normal - # distribution, so we use a rational approximation instead: - # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the - # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 - # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. - q = p - 0.5 - - if fabs(q) <= 0.425: - r = 0.180625 - q * q - # Hash sum: 55.88319_28806_14901_4439 - num = (((((((2.50908_09287_30122_6727e+3 * r + - 3.34305_75583_58812_8105e+4) * r + - 6.72657_70927_00870_0853e+4) * r + - 4.59219_53931_54987_1457e+4) * r + - 1.37316_93765_50946_1125e+4) * r + - 1.97159_09503_06551_4427e+3) * r + - 1.33141_66789_17843_7745e+2) * r + - 3.38713_28727_96366_6080e+0) * q - den = (((((((5.22649_52788_52854_5610e+3 * r + - 2.87290_85735_72194_2674e+4) * r + - 3.93078_95800_09271_0610e+4) * r + - 2.12137_94301_58659_5867e+4) * r + - 5.39419_60214_24751_1077e+3) * r + - 6.87187_00749_20579_0830e+2) * r + - 4.23133_30701_60091_1252e+1) * r + - 1.0) - x = num / den - return mu + (x * sigma) - - r = p if q <= 0.0 else 1.0 - p - r = sqrt(-log(r)) - if r <= 5.0: - r = r - 1.6 - # Hash sum: 49.33206_50330_16102_89036 - num = (((((((7.74545_01427_83414_07640e-4 * r + - 2.27238_44989_26918_45833e-2) * r + - 2.41780_72517_74506_11770e-1) * r + - 1.27045_82524_52368_38258e+0) * r + - 3.64784_83247_63204_60504e+0) * r + - 5.76949_72214_60691_40550e+0) * r + - 4.63033_78461_56545_29590e+0) * r + - 1.42343_71107_49683_57734e+0) - den = (((((((1.05075_00716_44416_84324e-9 * r + - 5.47593_80849_95344_94600e-4) * r + - 1.51986_66563_61645_71966e-2) * r + - 1.48103_97642_74800_74590e-1) * r + - 6.89767_33498_51000_04550e-1) * r + - 1.67638_48301_83803_84940e+0) * r + - 2.05319_16266_37758_82187e+0) * r + - 1.0) - else: - r = r - 5.0 - # Hash sum: 47.52583_31754_92896_71629 - num = (((((((2.01033_43992_92288_13265e-7 * r + - 2.71155_55687_43487_57815e-5) * r + - 1.24266_09473_88078_43860e-3) * r + - 2.65321_89526_57612_30930e-2) * r + - 2.96560_57182_85048_91230e-1) * r + - 1.78482_65399_17291_33580e+0) * r + - 5.46378_49111_64114_36990e+0) * r + - 6.65790_46435_01103_77720e+0) - den = (((((((2.04426_31033_89939_78564e-15 * r + - 1.42151_17583_16445_88870e-7) * r + - 1.84631_83175_10054_68180e-5) * r + - 7.86869_13114_56132_59100e-4) * r + - 1.48753_61290_85061_48525e-2) * r + - 1.36929_88092_27358_05310e-1) * r + - 5.99832_20655_58879_37690e-1) * r + - 1.0) - - x = num / den - if q < 0.0: - x = -x - - return mu + (x * sigma) - - -# If available, use C implementation -try: - from _statistics import _normal_dist_inv_cdf -except ImportError: - pass - - class NormalDist: "Normal distribution of a random variable" # https://en.wikipedia.org/wiki/Normal_distribution @@ -1555,11 +1478,13 @@ def _sum(data): types_add = types.add partials = {} partials_get = partials.get + for typ, values in groupby(data, type): types_add(typ) for n, d in map(_exact_ratio, values): count += 1 partials[d] = partials_get(d, 0) + n + if None in partials: # The sum will be a NAN or INF. We can ignore all the finite # partials, and just look at this special one. @@ -1568,6 +1493,7 @@ def _sum(data): else: # Sum all the partial sums using builtin sum. total = sum(Fraction(n, d) for d, n in partials.items()) + T = reduce(_coerce, types, int) # or raise TypeError return (T, total, count) @@ -1590,6 +1516,7 @@ def _ss(data, c=None): types_add = types.add sx_partials = defaultdict(int) sxx_partials = defaultdict(int) + for typ, values in groupby(data, type): types_add(typ) for n, d in map(_exact_ratio, values): @@ -1599,11 +1526,13 @@ def _ss(data, c=None): if not count: ssd = c = Fraction(0) + elif None in sx_partials: # The sum will be a NAN or INF. We can ignore all the finite # partials, and just look at this special one. ssd = c = sx_partials[None] assert not _isfinite(ssd) + else: sx = sum(Fraction(n, d) for d, n in sx_partials.items()) sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) @@ -1687,8 +1616,10 @@ def _convert(value, T): # This covers the cases where T is Fraction, or where value is # a NAN or INF (Decimal or float). return value + if issubclass(T, int) and value.denominator != 1: T = float + try: # FIXME: what do we do if this overflows? return T(value) @@ -1851,3 +1782,88 @@ def _sqrtprod(x: float, y: float) -> float: # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 d = sumprod((x, h), (y, -h)) return h + d / (2.0 * h) + + +def _normal_dist_inv_cdf(p, mu, sigma): + # There is no closed-form solution to the inverse CDF for the normal + # distribution, so we use a rational approximation instead: + # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the + # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 + # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. + q = p - 0.5 + + if fabs(q) <= 0.425: + r = 0.180625 - q * q + # Hash sum: 55.88319_28806_14901_4439 + num = (((((((2.50908_09287_30122_6727e+3 * r + + 3.34305_75583_58812_8105e+4) * r + + 6.72657_70927_00870_0853e+4) * r + + 4.59219_53931_54987_1457e+4) * r + + 1.37316_93765_50946_1125e+4) * r + + 1.97159_09503_06551_4427e+3) * r + + 1.33141_66789_17843_7745e+2) * r + + 3.38713_28727_96366_6080e+0) * q + den = (((((((5.22649_52788_52854_5610e+3 * r + + 2.87290_85735_72194_2674e+4) * r + + 3.93078_95800_09271_0610e+4) * r + + 2.12137_94301_58659_5867e+4) * r + + 5.39419_60214_24751_1077e+3) * r + + 6.87187_00749_20579_0830e+2) * r + + 4.23133_30701_60091_1252e+1) * r + + 1.0) + x = num / den + return mu + (x * sigma) + + r = p if q <= 0.0 else 1.0 - p + r = sqrt(-log(r)) + if r <= 5.0: + r = r - 1.6 + # Hash sum: 49.33206_50330_16102_89036 + num = (((((((7.74545_01427_83414_07640e-4 * r + + 2.27238_44989_26918_45833e-2) * r + + 2.41780_72517_74506_11770e-1) * r + + 1.27045_82524_52368_38258e+0) * r + + 3.64784_83247_63204_60504e+0) * r + + 5.76949_72214_60691_40550e+0) * r + + 4.63033_78461_56545_29590e+0) * r + + 1.42343_71107_49683_57734e+0) + den = (((((((1.05075_00716_44416_84324e-9 * r + + 5.47593_80849_95344_94600e-4) * r + + 1.51986_66563_61645_71966e-2) * r + + 1.48103_97642_74800_74590e-1) * r + + 6.89767_33498_51000_04550e-1) * r + + 1.67638_48301_83803_84940e+0) * r + + 2.05319_16266_37758_82187e+0) * r + + 1.0) + else: + r = r - 5.0 + # Hash sum: 47.52583_31754_92896_71629 + num = (((((((2.01033_43992_92288_13265e-7 * r + + 2.71155_55687_43487_57815e-5) * r + + 1.24266_09473_88078_43860e-3) * r + + 2.65321_89526_57612_30930e-2) * r + + 2.96560_57182_85048_91230e-1) * r + + 1.78482_65399_17291_33580e+0) * r + + 5.46378_49111_64114_36990e+0) * r + + 6.65790_46435_01103_77720e+0) + den = (((((((2.04426_31033_89939_78564e-15 * r + + 1.42151_17583_16445_88870e-7) * r + + 1.84631_83175_10054_68180e-5) * r + + 7.86869_13114_56132_59100e-4) * r + + 1.48753_61290_85061_48525e-2) * r + + 1.36929_88092_27358_05310e-1) * r + + 5.99832_20655_58879_37690e-1) * r + + 1.0) + + x = num / den + if q < 0.0: + x = -x + + return mu + (x * sigma) + + +# If available, use C implementation +try: + from _statistics import _normal_dist_inv_cdf +except ImportError: + pass diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4b3a0645cfc84a..a059a6b8340448 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5553,15 +5553,29 @@ def test_set_get(self): multiprocessing.set_start_method(old_method, force=True) self.assertGreaterEqual(count, 1) - def test_get_all(self): + def test_get_all_start_methods(self): methods = multiprocessing.get_all_start_methods() + self.assertIn('spawn', methods) if sys.platform == 'win32': self.assertEqual(methods, ['spawn']) + elif sys.platform == 'darwin': + self.assertEqual(methods[0], 'spawn') # The default is first. + # Whether these work or not, they remain available on macOS. + self.assertIn('fork', methods) + self.assertIn('forkserver', methods) else: - self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['spawn', 'fork'] or - methods == ['fork', 'spawn', 'forkserver'] or - methods == ['spawn', 'fork', 'forkserver']) + # POSIX + self.assertIn('fork', methods) + if other_methods := set(methods) - {'fork', 'spawn'}: + # If there are more than those two, forkserver must be one. + self.assertEqual({'forkserver'}, other_methods) + # The default is the first method in the list. + self.assertIn(methods[0], {'forkserver', 'spawn'}, + msg='3.14+ default must not be fork') + if methods[0] == 'spawn': + # Confirm that the current default selection logic prefers + # forkserver vs spawn when available. + self.assertNotIn('forkserver', methods) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': diff --git a/Lib/test/certdata/allsans.pem b/Lib/test/certdata/allsans.pem index 02f2c2e6346ef8..f6a63b9219d246 100644 --- a/Lib/test/certdata/allsans.pem +++ b/Lib/test/certdata/allsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCczEVv5D2UDtn6 -DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj -jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt -07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP -Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 -330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr -GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA -WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK -4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAQKCAYAB -m29nxPNjod5Wm4xydWQYbZj/J0qkcyru/i1qpqyDbGa1sRNcg5A/A/8BPuPcWxhR -/hvwVeD5XX2/i2cnQuv6D3DQP1cSNCxQPanwzknP2k7IVqUmG0RDErPWuoDIhCnR -ljp0NPQsnj0fLhEkcbgG0xwx7KceUDigGsiTbatIvvBHGhQzrmTpqlVVdtMWvGRt -HQEJYuMuIw6IwALHyy3CITv5wh/Bec5OhNoFF8iUZceR4ZkGWf8bYWIa25xlzH6K -4rhOOh1G2ObHHTjhZq4mGXTHY1MEkAxXKWlR3DJc0Lh5E1UETSI6WBHWRb08iwQ5 -AkLOPyMpt08xHFWbJqywvlxenpri+gjY3xbXqGNhyDYWHZqlQmJVnzxoUOuuHi2R -dO86IckUc4Thjbm7a6AilL9t8juNZvyeQUVgtVi25uPkm/cK6r5vo8y4UOUU41EN -NOathlF69gh93il4t6zJW9jPV2WENv1H/vhKUWKW6cabX3vy4rANwy3q4V//GDEC -gcEAuniGCHaEdSjV2sUHyt/yrCLbU6+eLTfNk6AQyXJk6Wrvj6g3gx90ewEq5i/C -ukmSKDslr9pupq8Z/UNfYHZfJfpwEsYvIZ8DdFSd62/h66DhIoEn1v3Lwt+aexgX -yGJHF0BG9JA2CU5Z5NGjlnQYqQBobO9uZMq62l15Ig1MAMHGL0ZYVvOqGZD7XvtC -4UnclK5kjp51Vd5rydEQxyi5qkyLl9Q6T0FQXOphGIOd8ifYeUGe7YC72cFPevdx -wDXZAoHBANdDVvCMrjmzdrS6td39/2nHTeerFPbsOe2LIQYzqjeEe6GWqd2NL9NZ -bk3/cAuVgbWtdvSQQhhmSqOC7JZic4hbZb3lK6v/sr4F/Zu0CfAu80swWFMeS7vq -eQeYzN4w4dKpJArvU3ll7N9AlZhdlYkbPf0WdeOIjZawdAOxNtNe0O+j+5MsXR59 -qkULatumhcKUnqxFCiVHzy21CVJtRzrtu6oGoSdFbmG82eSJ1rPXiuuDnCyzjyMW -iClYRM4NOQKBwERnO/vUxihYT4LOLlqcpl/A9aYQUT0TMGWMHTxYq2343WJceeiu -3ELXHc6NDKjbnjMF54BH57lbmHQQh+dR5PuAkCZC7z0tIM5G0Bty0nRmcs/+gwfZ -2Cpnbjrjjq3iZ2O/H4hNcpUdWdqXkKP7eKReUvBLMLrmp369NVdpe0z3yGTFMFjN -T8PLLHsePt14A+PCyX6L4E0cp3vEJpx4cwtmwvpyTuWN9xXuoKmmdoVDWqS4jr1f -MQnjYO2h4ed5mQKBwGVttWli4DUP+r7tuwP+ynptDqg6VIaEiEcFZ2okre+63QYm -l6NtAzvyx6a41XKf355bPdG+p2YXzNN+vTue6BE3/5iagxloQjCHYhgbnRMvDDRB -c1y2ybihoqWRufZ30fARAoqkehCZliMbq2E/t1YDIBJAowuzLAP04LVcqxitdIV2 -HvQZ00aqr7AY0SDuNdiZbqp9XWpzi4td4iaUlxuNKP/UX9rBPGGROpoU2LWkujB+ -svfdI3TFCSNyE/mDAQKBwQCP++WZKxExrSFRk3W+TcHKHZb2pusfoPWE7WH6EnDW -dkTZpa3PZaf0xgeglmNBv4Paxw2eMPsIhyNv62XY/6GbY6VJWRyx/s+NsazeP4ji -xUOufnwTePjYw6x0pcl6BknZrHn8LCJU741h0yTum8cDdNfRKdc0AMy0gVXk4ZTG -2cAtbEcWb3J+a5kYf6mp5yx3BNwtewkGZhc2VuQ9mQNbMmOOS/pHQQTRWcxsQwyt -GPAhMKawjrL1KFmu7vIqDSw= +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCYEY+lIwS/w+gA +U5PfxSeL4qMF5TtFkrifL0oQaCjWFb8jeX2Rb1tKaObyrYmTURVWbMkI1LJOQWYt +GlANo2mvf4VD1wZZ/x/Tmylxe3A7itSTZgZuGXLHtFP9wk7yS46BlVcNq3ZoOi7k +k8NxTiUqgsmLg3Nj+Ifk8OI7c1BcP2pa1wZO/l3PTWqtFSLsFEvOK5N6ZPqILAUi +5vEkOiFefxVHHvD4N3IWSFgSwiDi9KhntdRwa+axsZxjq/o70FjQO5q5Quym9aDq +qo47PMykEb1gdTBQK1mPliMKsLi5zycI/ypm1PEcH53+9/F+U2B1bmvA/9PBx8T7 +hUyclBbQHCmdcXIYCgFr5AbGS3jFeVPiF7hCPj1k/jcpayXZAtWmqJTlkXKGbM68 +AgM/DHO0fGCYGRUw2vhV3ZIRCm0BqwforeFh+2oMyZjDjvo/uZjZNazdpcwvOy0c +lFkORwVI0zTIixXYSV4xN1BX821JCuhYubro89m+MSjxO14qgs0CAwEAAQKCAYAZ +txqF7+qNLSlN6NRASnw6BQzRYebIiJus25fYN2z0awAEFTbdQan75sprLlpt7Y/A +qivC0QkH/7oyFVmFPOWR1mwoQTPjQyfUJlB3Tsr3Xq488MtUkfwddkqfxlyIT6ud +ES6D8sNWs8QbRjuOLQtO6YgAji2UscH1PqDbMdBckSLAks0Pzab6d9p6w3DA4FvD +VQ4e6/WL0nnZ4ZjUqfnbm3zzJnHUX7fsubYfEfHyvzG9O/vdOPntgZ3zIvFxbPVc +/Dp8MRNgfQoRgAsc7VBQQifFNvva9SjvRsXwE7zQLQh9JszFpHWqIMevGwjdH8dO +1hHESqkRHL0sYwsqoGqbaHFrJVEA2Ljuxs/UsH6G2U6NuDVaYkqM8UwH0+EqGFtF +9qP8oh6Yvp+szI5yET4brW0Rlk8r2UqvQ5iSSl8SSWsNKR2wRs0PM0aNm13TDJmt +NOuxd/0ranG+NvBMdtLCwsoEplYDaaczYv9gbKHT64OlFAdGBkOytbS7eqOojmcC +gcEA0uQb9bPKbiMYSEJMAalOlruzvMiHBRW420IEuGyHJtocn/RK804Jj6gzu+WD +GT81rB9NXAt/3s6WDMSdrGrrBh/CQZJs50ok9hej0f6gsq1JNsWJ/c10rU/mqrTn +s++pdaRqbd+bgM/N6h79tJi3j4SgB9rJqjC+7qWlDJFWZOoDR59elc10xDA0AE0a +K6UvsJJGk4/v715K1Uh2qU5HYupuijNc0m3z2OMZ6zxTNNUdbSL4XF5X9e2+BwMQ +YrRzAoHBALiYeYWsmHGRdg++/p/9SE86YwyEYr+zrUiWenIPC1W5WQlkdFQqALmg +kjcXAXFFouC/DnKG65YOfGq772ekFuw06Lm4DEotQP02pESfAP9ulVufK2yJtbxi +m0PeN3JRrs/H72VW9hUSxCafNRyBeZ6nsfzEfEkPOdc+lRGeMHT9C7r7Y1asNk8A +Zl7wKZgv+Yvn8xhHWQj5z/T6K2z5CXFn6FERc0qV8+DggE9m3CjaaNxft6IodfHD +h1iakNtbvwKBwQCpWr/NRy131r0IQh0xdFoFGAUVxF8RSUli4hhSVe0O2TcFiLOr +wW5SK/wnlv75hlY+vABuu1lbfsDmzfnk3RORnm1sJP9JmbQm4AMRfw5jjl7uGiJf +a9+X0kNlsNMlH4ARVhCV3WzOO5KbwXlxzvYRzaqJxDwQbQbXNLRfbFNZxMcPfD8D +w7NSXXdVCpXKmOO8QytkEsHWkv07W+7WtWMEX0iXuPmAjwW0lWNaEd6r3by8yMlz +u9udRedFUEOXUFsCgcAX0g00c75EQXoTtBjVenC/UJCBh//aLwx4Znqsh0Z2LHHR +5XWher4XNiJIG57jCBJpoB30J3b1KS9i8peFL0aJ+pXhiV+EnuxZAJkYBdCyJYn+ +hb6rxeV+xta0XlOXW/UL+QfqcttUgtRvC3JmGEsibw9nx88l+mIDZZ8E4/3qytCd +s1zxTU3AyhNrwuALNH2mUSssgeB6aQot2a6K5GQUj00KUQ8om8sZxL6qAGL+npiT +f4KJ2WDG7u1jQKbat68CgcBONTGp3M3pdwc3OV/PMFq1+h1/BuyW/5qUtbsvXkrD +DZyXGY2SxJFMipakzOppusaQOYmO+VXKVDRJm8UG6fa2cdXjwm2kDxHB5E+v59VI ++uM8ErX6L52I0+rdoOU6AwpXyiVW5AXzMInOGHF4B7zJ1SA25RwVG44LbDw1cehW +MALXUdVCaegmvCCFq5TQlrlaEktxWk6Th92mXWbNbwUHlIGpcjEkASFvt90aBXWZ +w0YCQFVxx/K95nzKyRTjHTQ= -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=allsans Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:9c:cc:45:6f:e4:3d:94:0e:d9:fa:0c:c9:99:fe: - e0:96:08:bc:8b:f8:ae:71:4d:9a:7f:e6:3d:dc:c8: - 84:a8:69:3b:94:d4:ab:b7:90:3f:7b:78:b8:29:1e: - 3e:05:82:4e:59:26:2d:b5:ce:c4:2a:7a:90:a3:8c: - 08:41:93:ab:63:e9:16:14:72:73:1b:6f:c3:2e:c5: - 49:10:30:40:d6:fb:92:df:4b:2a:7e:1c:26:9e:b6: - 51:e8:b1:67:61:10:3a:97:e2:4a:49:e5:a8:c1:c0: - d4:6d:d3:b9:b1:40:4c:6d:0b:89:1c:9e:8d:2e:b7: - da:c6:e5:7c:cb:37:6f:57:a4:2a:51:1e:51:45:ae: - 7b:4e:81:00:04:a2:2c:fd:fd:f7:3c:a9:a4:e0:6f: - 7e:d1:80:66:0f:42:cc:61:ec:f7:11:0d:17:b8:64: - 6c:75:97:4e:88:ae:bf:18:fb:65:b9:d3:35:82:7e: - 4c:02:4b:70:da:2c:33:8a:22:34:8e:7c:f1:cd:aa: - 67:dc:7e:c4:e9:ce:b8:fc:df:7d:1c:74:f9:25:ac: - c2:2e:88:1b:fe:88:69:25:0a:39:0f:64:98:84:87: - 1b:d8:da:4e:84:44:ce:07:03:1c:6e:3c:28:af:ac: - 7e:4c:09:e3:0d:62:f0:55:51:c2:eb:19:13:a6:ec: - 94:57:46:59:48:fd:7b:9b:c3:0a:00:68:06:fe:cb: - e7:59:b4:26:7d:02:7c:c7:21:bf:a7:c8:a8:89:c5: - 60:23:ce:34:7e:40:f2:fa:8c:7a:d5:58:15:40:59: - 37:3c:dd:83:f9:16:fc:54:5c:c7:d8:de:c5:6f:f6: - d3:52:09:5d:dc:a5:83:96:e7:e8:97:71:0c:cb:b9: - cc:d7:9c:e2:d1:62:4e:e9:f7:09:1b:d3:f8:64:f4: - ab:8a:e0:13:00:f7:47:81:86:f1:4c:08:58:07:27: - 5f:f1:26:0d:a7:16:60:47:3e:8f:f1:6d:cc:0f:1b: - f9:9a:e5:3f:cc:70:0f:26:02:51 + 00:98:11:8f:a5:23:04:bf:c3:e8:00:53:93:df:c5: + 27:8b:e2:a3:05:e5:3b:45:92:b8:9f:2f:4a:10:68: + 28:d6:15:bf:23:79:7d:91:6f:5b:4a:68:e6:f2:ad: + 89:93:51:15:56:6c:c9:08:d4:b2:4e:41:66:2d:1a: + 50:0d:a3:69:af:7f:85:43:d7:06:59:ff:1f:d3:9b: + 29:71:7b:70:3b:8a:d4:93:66:06:6e:19:72:c7:b4: + 53:fd:c2:4e:f2:4b:8e:81:95:57:0d:ab:76:68:3a: + 2e:e4:93:c3:71:4e:25:2a:82:c9:8b:83:73:63:f8: + 87:e4:f0:e2:3b:73:50:5c:3f:6a:5a:d7:06:4e:fe: + 5d:cf:4d:6a:ad:15:22:ec:14:4b:ce:2b:93:7a:64: + fa:88:2c:05:22:e6:f1:24:3a:21:5e:7f:15:47:1e: + f0:f8:37:72:16:48:58:12:c2:20:e2:f4:a8:67:b5: + d4:70:6b:e6:b1:b1:9c:63:ab:fa:3b:d0:58:d0:3b: + 9a:b9:42:ec:a6:f5:a0:ea:aa:8e:3b:3c:cc:a4:11: + bd:60:75:30:50:2b:59:8f:96:23:0a:b0:b8:b9:cf: + 27:08:ff:2a:66:d4:f1:1c:1f:9d:fe:f7:f1:7e:53: + 60:75:6e:6b:c0:ff:d3:c1:c7:c4:fb:85:4c:9c:94: + 16:d0:1c:29:9d:71:72:18:0a:01:6b:e4:06:c6:4b: + 78:c5:79:53:e2:17:b8:42:3e:3d:64:fe:37:29:6b: + 25:d9:02:d5:a6:a8:94:e5:91:72:86:6c:ce:bc:02: + 03:3f:0c:73:b4:7c:60:98:19:15:30:da:f8:55:dd: + 92:11:0a:6d:01:ab:07:e8:ad:e1:61:fb:6a:0c:c9: + 98:c3:8e:fa:3f:b9:98:d9:35:ac:dd:a5:cc:2f:3b: + 2d:1c:94:59:0e:47:05:48:d3:34:c8:8b:15:d8:49: + 5e:31:37:50:57:f3:6d:49:0a:e8:58:b9:ba:e8:f3: + d9:be:31:28:f1:3b:5e:2a:82:cd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 31:5E:C0:5E:2F:47:FF:8B:92:F9:EE:3D:B1:87:D0:53:75:3B:B1:48 + 73:0D:4E:1F:4A:EC:F2:53:0F:53:FC:85:6F:CF:82:CD:A3:E8:11:8D X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,65 +103,65 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 72:42:a6:fc:ee:3c:21:47:05:33:e8:8c:6b:27:07:4a:ed:e2: - 81:47:96:79:43:ff:0f:ef:5a:06:aa:4c:01:70:5b:21:c4:b7: - 5d:17:29:c8:10:02:c3:08:7b:8c:86:56:9e:e9:7c:6e:a8:b6: - 26:13:9e:1e:1f:93:66:85:67:63:9e:08:fb:55:39:56:82:f5: - be:0c:38:1e:eb:c4:54:b2:a7:7b:18:55:bb:00:87:43:50:50: - bb:e1:29:10:cf:3d:c9:07:c7:d2:5d:b6:45:68:1f:d6:de:00: - 96:3e:29:73:f6:22:70:21:a2:ba:68:28:94:ec:37:bc:a7:00: - 70:58:4e:d1:48:ae:ef:8d:11:a4:6e:10:2f:92:83:07:e2:76: - ac:bf:4f:bb:d6:9f:47:9e:a4:02:03:16:f8:a8:0a:3d:67:17: - 31:44:0e:68:d0:d3:24:d5:e7:bf:67:30:8f:88:97:92:0a:1e: - d7:74:df:7e:7b:4c:c6:d9:c3:84:92:2b:a0:89:11:08:4c:dd: - 32:49:df:36:23:d4:63:56:e4:f1:68:5a:6f:a0:c3:3c:e2:36: - ee:f3:46:60:78:4d:76:a5:5a:4a:61:c6:f8:ae:18:68:c2:8d: - 0e:2f:76:50:bb:be:b9:56:f1:04:5c:ac:ad:d7:d6:a4:1e:45: - 45:52:f4:10:a2:0f:9b:e3:d9:73:17:b6:52:42:a6:5b:c9:e9: - 8d:60:74:68:d0:1f:7a:ce:01:8e:9e:55:cb:cf:64:c1:cc:9a: - 72:aa:b4:5f:b5:55:13:41:10:51:a0:2c:a5:5b:43:12:ca:cc: - b7:c4:ac:f2:6f:72:fd:0d:50:6a:d6:81:c1:91:93:21:fe:de: - 9a:be:e5:3c:2a:98:95:a1:42:f8:f2:5c:75:c6:f1:fd:11:b1: - 22:26:33:5b:43:63:21:06:61:d2:cd:04:f3:30:c6:a8:3f:17: - d3:05:a3:87:45:2e:52:1e:51:88:e3:59:4c:78:51:b0:7b:b4: - 58:d9:27:22:6e:8c + a5:3d:12:87:b9:3b:3e:9c:1c:59:fb:d5:38:22:49:61:f3:c3: + 11:53:4b:4e:63:af:f2:3d:ef:24:67:45:bc:74:5c:4a:65:c5: + b4:bb:fe:84:b8:b6:ca:7d:fc:aa:ff:80:ae:67:1f:cb:c3:cd: + 8f:f9:75:8c:f9:d3:3f:db:f6:81:d8:06:42:c3:5d:a9:1e:a3: + 81:7d:57:ac:97:d9:bd:c8:ce:1e:ec:74:d7:94:d5:df:b1:ad: + ce:84:14:5c:8c:45:a4:a8:eb:67:ab:16:57:61:15:86:ae:11: + 1e:b5:10:42:de:84:76:88:9b:37:12:aa:a6:77:42:75:b4:c0: + 04:b3:75:45:e0:d7:aa:34:e3:07:c5:ed:f8:4e:f0:39:99:1f: + 5b:d8:4e:0c:ad:64:6d:09:07:3f:e3:e1:9f:1b:65:07:96:59: + 9a:b5:f1:4d:c3:ec:f7:32:a1:05:94:d1:0b:18:54:3c:67:cf: + 38:f5:2b:ec:cb:bd:79:be:f7:1b:b7:71:3f:c6:44:80:7f:00: + dc:3d:a0:07:c0:b5:1c:fb:52:f6:a0:f8:92:c6:c6:73:07:c5: + ca:0b:04:7c:55:51:e3:ba:93:32:17:bd:61:ae:cf:13:e4:5e: + 03:b2:51:11:c7:68:f5:08:b6:0e:57:49:11:3c:e3:f4:0e:e1: + 96:20:44:28:31:94:11:44:50:cf:17:70:8d:9c:14:c5:ed:94: + 4d:ba:94:9b:db:8b:9f:55:a1:5e:0a:90:bb:a0:0e:0d:3b:a0: + dd:4d:47:d1:cf:d0:47:6b:ff:6f:af:e4:83:40:73:e6:3a:59: + 40:bd:4f:3a:21:22:63:27:5d:02:26:67:89:1d:2f:19:c5:77: + e6:b5:67:4a:dd:b5:0e:de:46:34:57:c7:03:84:5d:cd:34:08: + 8f:47:c9:4d:7f:04:c0:4f:ff:68:52:bb:ae:84:0e:54:ce:5c: + 27:71:0f:a2:3f:9f:7a:49:a8:fa:0a:45:cf:96:42:a7:65:23: + b9:3e:40:eb:46:7b -----BEGIN CERTIFICATE----- -MIIHDTCCBXWgAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIHDzCCBXegAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2Fs -bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCczEVv5D2UDtn6 -DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj -jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt -07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP -Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 -330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr -GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA -WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK -4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAaOCAt4w -ggLaMIIBMAYDVR0RBIIBJzCCASOCB2FsbHNhbnOgHgYDKgMEoBcMFXNvbWUgb3Ro -ZXIgaWRlbnRpZmllcqA1BgYrBgEFAgKgKzApoBAbDktFUkJFUk9TLlJFQUxNoRUw -E6ADAgEBoQwwChsIdXNlcm5hbWWBEHVzZXJAZXhhbXBsZS5vcmeCD3d3dy5leGFt -cGxlLm9yZ6RnMGUxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJh -eDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xGDAWBgNVBAMM -D2Rpcm5hbWUgZXhhbXBsZYYXaHR0cHM6Ly93d3cucHl0aG9uLm9yZy+HBH8AAAGH -EAAAAAAAAAAAAAAAAAAAAAGIBCoDBAUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW -MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQx -XsBeL0f/i5L57j2xh9BTdTuxSDB9BgNVHSMEdjB0gBTz7JSO8o4wxI5owr+OahnA -wZ92ZaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy -ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp -UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w -eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo -dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E -PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y -ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAckKm/O48IUcFM+iMaycH -Su3igUeWeUP/D+9aBqpMAXBbIcS3XRcpyBACwwh7jIZWnul8bqi2JhOeHh+TZoVn -Y54I+1U5VoL1vgw4HuvEVLKnexhVuwCHQ1BQu+EpEM89yQfH0l22RWgf1t4Alj4p -c/YicCGiumgolOw3vKcAcFhO0Uiu740RpG4QL5KDB+J2rL9Pu9afR56kAgMW+KgK -PWcXMUQOaNDTJNXnv2cwj4iXkgoe13TffntMxtnDhJIroIkRCEzdMknfNiPUY1bk -8Whab6DDPOI27vNGYHhNdqVaSmHG+K4YaMKNDi92ULu+uVbxBFysrdfWpB5FRVL0 -EKIPm+PZcxe2UkKmW8npjWB0aNAfes4Bjp5Vy89kwcyacqq0X7VVE0EQUaAspVtD -EsrMt8Ss8m9y/Q1QataBwZGTIf7emr7lPCqYlaFC+PJcdcbx/RGxIiYzW0NjIQZh -0s0E8zDGqD8X0wWjh0UuUh5RiONZTHhRsHu0WNknIm6M +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +YWxsc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAJgRj6UjBL/D +6ABTk9/FJ4viowXlO0WSuJ8vShBoKNYVvyN5fZFvW0po5vKtiZNRFVZsyQjUsk5B +Zi0aUA2jaa9/hUPXBln/H9ObKXF7cDuK1JNmBm4Zcse0U/3CTvJLjoGVVw2rdmg6 +LuSTw3FOJSqCyYuDc2P4h+Tw4jtzUFw/alrXBk7+Xc9Naq0VIuwUS84rk3pk+ogs +BSLm8SQ6IV5/FUce8Pg3chZIWBLCIOL0qGe11HBr5rGxnGOr+jvQWNA7mrlC7Kb1 +oOqqjjs8zKQRvWB1MFArWY+WIwqwuLnPJwj/KmbU8Rwfnf738X5TYHVua8D/08HH +xPuFTJyUFtAcKZ1xchgKAWvkBsZLeMV5U+IXuEI+PWT+NylrJdkC1aaolOWRcoZs +zrwCAz8Mc7R8YJgZFTDa+FXdkhEKbQGrB+it4WH7agzJmMOO+j+5mNk1rN2lzC87 +LRyUWQ5HBUjTNMiLFdhJXjE3UFfzbUkK6Fi5uujz2b4xKPE7XiqCzQIDAQABo4IC +3jCCAtowggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBv +dGhlciBpZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2h +FTAToAMCAQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4 +YW1wbGUub3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRo +cmF4MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UE +AwwPZGlybmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAA +AYcQAAAAAAAAAAAAAAAAAAAAAYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE +FHMNTh9K7PJTD1P8hW/Pgs2j6BGNMH0GA1UdIwR2MHSAFMAKKyhD3l/JfUflR5s2 +8mWMZzvioVGkTzBNMQswCQYDVQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3 +YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQDLLYCZ +WmlSWzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNh +LnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGG +KWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1Ud +HwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNh +L3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBCwUAA4IBgQClPRKHuTs+nBxZ+9U4 +Iklh88MRU0tOY6/yPe8kZ0W8dFxKZcW0u/6EuLbKffyq/4CuZx/Lw82P+XWM+dM/ +2/aB2AZCw12pHqOBfVesl9m9yM4e7HTXlNXfsa3OhBRcjEWkqOtnqxZXYRWGrhEe +tRBC3oR2iJs3Eqqmd0J1tMAEs3VF4NeqNOMHxe34TvA5mR9b2E4MrWRtCQc/4+Gf +G2UHllmatfFNw+z3MqEFlNELGFQ8Z8849Svsy715vvcbt3E/xkSAfwDcPaAHwLUc ++1L2oPiSxsZzB8XKCwR8VVHjupMyF71hrs8T5F4DslERx2j1CLYOV0kRPOP0DuGW +IEQoMZQRRFDPF3CNnBTF7ZRNupSb24ufVaFeCpC7oA4NO6DdTUfRz9BHa/9vr+SD +QHPmOllAvU86ISJjJ10CJmeJHS8ZxXfmtWdK3bUO3kY0V8cDhF3NNAiPR8lNfwTA +T/9oUruuhA5UzlwncQ+iP596Saj6CkXPlkKnZSO5PkDrRns= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 index aa9dbbe294f829..6d773ee5b32432 100644 --- a/Lib/test/certdata/capath/b1930218.0 +++ b/Lib/test/certdata/capath/b1930218.0 @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 -CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh -Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 -VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o -eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT -NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 -0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB -D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J -d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO -MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD -ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO -S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn -QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ -rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd -2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 -tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y -kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 -SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO -aOZF0w== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 index aa9dbbe294f829..6d773ee5b32432 100644 --- a/Lib/test/certdata/capath/ceff1710.0 +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 -CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh -Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 -VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o -eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT -NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 -0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB -D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J -d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO -MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD -ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO -S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn -QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ -rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd -2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 -tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y -kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 -SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO -aOZF0w== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem index 034bc43ff1974e..a11dc614657d2f 100644 --- a/Lib/test/certdata/cert3.pem +++ b/Lib/test/certdata/cert3.pem @@ -1,34 +1,34 @@ -----BEGIN CERTIFICATE----- -MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt -3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY -IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh -P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd -pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW -9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo -TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o -8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa -IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB -wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ -jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst -gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw -AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD -VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN -JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 -+DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN -hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu -okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA -zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV -OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 -SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ -anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== ------END CERTIFICATE----- \ No newline at end of file +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem index 07a42422af1fd3..ebd7250cb4eae9 100644 --- a/Lib/test/certdata/idnsans.pem +++ b/Lib/test/certdata/idnsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQCp6zt40WB3K7yj -BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 -nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd -1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc -LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO -jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV -WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj -nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx -MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAQKCAYAz -9ZdHkDsf5fN2pAznXfOOOOz8+2hMjmwkn42GAp1gdWr+Z5GFiyaC8oTTSp6N1AnZ -iqCk8jcrHYMFi1JIOG8TzFjWBcGsinxsmp4vGDmvq2Ddcw5IiD2+rHJsdKZAOBP9 -snpD9cTE3zQYAu0XbE617krrxRqoSBO/1SExRjoIgzPCgFGyarBQl/DGjC/3Tku2 -y6oL4qxFqdTMD9QTzUuycUJlz5xu2+gaaaQ3hcMUe2xnZq28Qz3FKpf2ivZmZqWf -4+AIe0lRosmFoLAFjIyyuGCkWZ2t9KDIZV0OOS4+DvVOC/Um9r4VojeikripCGKY -2FzkkuQP3jz6pJ1UxCDg7YXZdR2IbcS18F1OYmLViU8oLDR6T01s0Npmp39dDazf -A4U+WyV3o1ydiSpwAiN8pJadmr5BHrCSmawV8ztW5yholufrO+FR5ze2+QG99txm -6l7lUI8Nz01lYG6D10MjaQ9INk2WSjBPVNbfsTl73/hR76/319ctfOINRMHilJ0C -gcEAvFgTdc5Qf9E7xEZE5UtpSTPvZ24DxQ7hmGW3pTs6+Nw4ccknnW0lLkWvY4Sf -gXl4TyTTUhe6PAN3UHzbjN2vdsTmAVUlnkNH40ymF8FgRGFNDuvuCZvf5ZWNddSl -6Vu/e5TFPDAqS8rGnl23DgXhlT+3Y0/mrftWoigdOxgximLAsmmqp/tANGi9jqE1 -3L0BN+xKqMMKSXkMtuzJHuini8MaWQgQcW/4czh4ArdesMzuVrstOD8947XHreY9 -pesVAoHBAOb0y/AFEdR+mhk/cfqjTzsNU2sS9yHlzMVgecF8BA26xcfAwg4d47VS -+LK8fV6KC4hHk4uQWjQzCG2PYXBbFT52xeJ3EC8DwWxJP09b4HV/1mWxXl5htjnr -dfyTmXKvEe5ZBpKGWc8i7s7jBi7R5EpgIfc586iNRyjYAk60dyG0iP13SurRvXBg -ID25VR4wABl3HQ3Hhv61dqC9FPrdHZQJdysfUqNrAFniWsSR2eyG5i4S1uHa3G+i -MzBTOuBRlwKBwBNXUBhG6YlWqTaMqMKLLfKwfKM4bvargost1uAG5xVrN/inWYQX -EzxfN5WWpvKa0Ln/5BuICD3ldTk0uS8MDNq7eYslfUl1S0qSMnQ6DXK4MzuXCsi9 -0w42f2JcRfVi0JUWP/LgV1eVKTRWF1g/Tl0PP/vY1q2DI/BfAjFxWJUHcxZfN4Es -kflP0Dd3YpqaZieiAkC2VrYY0i9uvXCJH7uAe5Is+9NKVk8uu1Q8FGM/iDIr4obm -J6rcnfbDsAz7yQKBwGtIbW9qO3UU9ioiQaTmtYg90XEclzXk1HEfNo+9NvjVuMfo -b3w1QDBbgXEtg6MlxuOgNBaRkIVM625ROzcA6GZir9tZ6Wede/z8LW+Ew0hxgLsu -YCLBiu9uxBj2y0HttwubySTJSfChToNGC/o1v7EY5M492kSCk/qSFMhQpkI+5Z+w -CVn44eHQlUl2zOY/79vka9eZxsiMrLVP/+3kRrgciYG7hByrOLeIIRfMlIl9xHDE -iZLSorEsjFC3aNMIswKBwFELC2fvlziW9rECQcGXnvc1DPmZcxm1ATFZ93FpKleF -TjLIWSdst0PmO8CSIuEZ2ZXPoK9CMJyQG+kt3k7IgZ1xKXg9y6ThwbznurXp1jaW -NjEnYtFMBK9Ur3oaAsrG2XwZ2PMvnI/Yp8tciGvjJlzSM8gHJ9BL8Yf+3gIJi/0D -KtaF9ha9J/SDDZdEiLIQ4LvSqYmlUgsCgiUvY3SVwCh8xDfBWD1hKw9vUiZu5cnJ -81hAHFgeD4f+C8fLols/sA== +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDeGGysqNqsz64+ +xsN+CUZDfCgPPhzUK0lDUlkEkp0ZeFeQm3+po2Y82gvJIValO5ChiXjPxSgJwoto +SHe4QZ8fWAlkX1d19A8xjjetx1K3dqZstqe3mPP7HQvfJlRkmAtVhFaUj1+SUx9K +s4HB4Lk2hqKUoq3TlRMpeSfJTXaHThVP7KZPWBzfM7pdpGYnznBpxgfymNrpTcXN +CjeR6lUlj/TExHiYLj4E0aODR/SUz0tBNdCQRcRDw+OyREe6Rk32Krl34Pr1dNKe +ER0tulhM2wRFsqw0a8vNs13I30BH4aXAr5bGgLCF2tXbP3KoeoR/GkQSWV14KrhT +xhKD3QEJ8gHjoHYKr4IdcFYcTakbI9E3THsQjQTUMJ/Xygza7OOr6pNPKY10tYfp +7WhyyY7cB/C/rHnUY0f58IZlRdMYgcLCUl7wzUxSVEMyr9Ary8XZZlxuO629k2ud +AIGwIi2TVq+UK0kNPZyGicGHMeEhoNc3YDMB/zYeqMr4KryI+mkCAwEAAQKCAYA4 +EENSnHdC+1P5bdRIew/dFjjIjD3bwyeF0oI9GMOGe+3ix5YE3QYAY2xpM7y7Dhu2 +40x3akXunMjzJKPwA8SmtWL9juG1mUvCjyt39yJmxJFDTSJuQrKIF694/6R7FjR6 +PGNcsgqGlewGv+SH6/HlFTxyN9SYXf/NztMfyimbAzd3Cv56dfwnzdeELu1IrCCN +WtuDvlk4XpUJasRXVadzyXCYwR3OEJJARik4CRBxBhjxl6OT38Co+IiAZiMTHw6z +jXeRuvyxXyYB76BQs9uCt05c5JV1I2OT1aKdpC0EbHlJEzJhmxrpdfBeoVdJUVVg +4b0QoocZE89KXOVmjmIKExxmjOjxH6o6qfxvDQWLspADD4D+69Zqjj6hFAfn3EUW +hcq3RHwuUheCE6i2xRtWPnA1ygqWN9mdNojX1+EXIv0Dh29kFWHpMUC29WWm9JA6 +Arx2UrtvoWGYotKxTDxZ0SV5ws/LzGjvMaT/dbXZHjywnB7sPfpcE9b91p8Q6VcC +gcEA+Wr44ZzyMirF9FYYA+KEBL91khmffhXzCDkRXqd1j2BCQWETWqvrUuAx+kwh +FZ5AABdip96w8Eaj8XBWb1c81mJUbOimbcqiV6B4GokCI/7u3FkOMPFRNBq41R/M +43JH6QLHNxhXI4OlZsxv5WHmR5N88kXEHhofBoSn14EHtdkK/Hu8ly1Wux4/da83 +2wanwlysmQkk1CnpJVT0st7HrKHP8YWDbcBvH+xysHOIOs22q3gvAF2rsdORidpR +fZnHAoHBAOP03brYpLU1xa5LHQz/B7uWfgJm3l7wRQ627YOfUBe4IjnH2y/kNKjv +ROVjaooB2gtya6Z6xVuY1m4LzUfYuRN1FdX1Np4NjvfgPNFViX+0HoxyAytXn++V +Iy1tCpL6X43Iqrj/VxBcX09vIpbsprW2d3dcIo/45abkAO6Uoa6HUmZoqrqrBA7b +LnuhEfBAyyN42OiXbHDqif7XAWC30yY6k0FfDaOSLiBkAOKZ9lkf7ei3eNr4SdOa +Tf5a/RVKTwKBwB7YfOkiCM3tfkfGcffhBqSzrO2hn5jvS/wjWqOTIDXYGLmPMN6Q +zmyUb3nd+mV7Cb05JylNoCJHCjVsyDPC3TJCPOCvMQ349nTR0qitcwdSmuXDWb7x +yTIhb+Rjp2olkwEdJ9gHeZdZy5XYCKqcnecSNWyc9jEm19lthHhha7uwmOw6vUsQ +/13q0rxSLB05SHwADBRtDhHzEPNd+1k3tggChv3+ng9vsg6HpnNuBlYHZOT12xI3 +g2ldme0rg9J9twKBwA4R1gGrT3czy3C3iCJ+Ny732e0yBjWb5NdEqSI/mgTsw4gH +ctrg3fMzWXBDE5dTB+8+77AF0dqWc121csUldj7iMifTi7xzn8hi2b4d5m+wYVZP +zyxEq0VxUguCuG1b8Lvij879S5Vh7iwL8vmXv65lhbgjQqraNOp5FimjmNsZ1Rcn +DKqa1ZRJKPROe7n1ddRJqDGq7vGFOGE3Sgl7Lxgj82TMhh37bsdnBLr3v8G+e8Oq +V1ZEjuH1myzA1vASdwKBwQDcMHbUKeoJUxyIlABFsWxhm0MKwL5ZRgo7rhH01rF2 +TF3mbAEHHsAEfkHgBVRooifbxglxUDy1olIyRk+kqs1gmXZA0UmE9lUqoX5+n/j8 +pgbCc3sV53x7JJ4BCnAn49XTOnTi2ILeQ3MvTqcj+sDKo+0T4Zq0r5d8ZrMikKbO +HJ4MMBGth645QKhgpb0XgltvSn8aceS4uTxIKrbNpZVnWj/VzuOYoXQmKnZko9p1 +dbyjt6PMeVXWj0tz8FB2DGE= -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:a9:eb:3b:78:d1:60:77:2b:bc:a3:04:6b:a0:9d: - 1b:aa:23:70:29:7e:d4:e1:65:60:c8:a6:f5:b4:71: - 59:8d:d2:7a:90:c6:c3:ba:08:29:d2:e1:d4:bb:49: - 88:46:16:22:2a:5a:33:55:73:65:86:d2:0b:fb:9c: - 2e:43:16:0b:aa:6e:77:4d:66:4f:7f:4a:35:fc:10: - ec:4a:cf:80:0d:8d:de:b5:5a:74:e4:0c:ee:53:6e: - 28:b2:77:e4:ee:10:1c:61:99:b1:05:eb:d6:c5:f5: - db:5d:d5:28:28:f2:39:8c:ee:3f:9a:6b:a0:a5:62: - 1f:f8:f5:bc:7e:6e:52:43:17:be:84:19:2a:e6:07: - 5a:be:a1:b3:1e:65:44:61:3d:73:41:7a:88:aa:29: - 1f:37:c7:91:dc:2d:32:58:32:87:2f:6f:a9:bc:b8: - 00:70:b9:29:1a:d7:71:9b:dd:01:ae:d3:aa:79:95: - d2:ba:4c:32:b7:f0:7b:28:3b:2e:8c:14:ee:45:d4: - fa:e4:6a:41:bf:78:11:ce:8d:c0:4e:18:f7:cd:99: - 3c:16:3a:ab:f0:95:c4:16:ca:4a:f8:42:ca:56:4b: - ec:93:cd:d9:20:67:e9:b1:6a:bf:f2:42:ea:bb:c2: - e6:12:35:a1:1f:18:7e:72:a1:8b:55:58:88:0a:c3: - 06:51:0c:fa:a9:49:0b:95:7b:e2:7e:31:62:cf:43: - a4:53:33:ea:d7:20:75:73:1e:d3:e4:d5:ff:03:ae: - 03:7e:85:27:a9:27:2f:ec:54:10:44:40:2c:e3:9f: - 20:7e:b1:c9:fb:e8:44:d6:72:1d:e2:c9:5a:0c:8f: - 7a:15:38:6b:34:6f:85:d3:0c:c9:94:c3:f0:e6:12: - de:53:0d:a8:7b:4e:a1:37:1b:e7:e9:04:86:e0:31: - 28:f1:33:35:10:66:5d:bb:8b:39:8c:f6:74:02:25: - 10:28:b0:da:0d:c1:7a:a5:33:be:78:2b:d5:9c:63: - a8:13:f2:49:1c:8f:50:9f:4e:63 + 00:de:18:6c:ac:a8:da:ac:cf:ae:3e:c6:c3:7e:09: + 46:43:7c:28:0f:3e:1c:d4:2b:49:43:52:59:04:92: + 9d:19:78:57:90:9b:7f:a9:a3:66:3c:da:0b:c9:21: + 56:a5:3b:90:a1:89:78:cf:c5:28:09:c2:8b:68:48: + 77:b8:41:9f:1f:58:09:64:5f:57:75:f4:0f:31:8e: + 37:ad:c7:52:b7:76:a6:6c:b6:a7:b7:98:f3:fb:1d: + 0b:df:26:54:64:98:0b:55:84:56:94:8f:5f:92:53: + 1f:4a:b3:81:c1:e0:b9:36:86:a2:94:a2:ad:d3:95: + 13:29:79:27:c9:4d:76:87:4e:15:4f:ec:a6:4f:58: + 1c:df:33:ba:5d:a4:66:27:ce:70:69:c6:07:f2:98: + da:e9:4d:c5:cd:0a:37:91:ea:55:25:8f:f4:c4:c4: + 78:98:2e:3e:04:d1:a3:83:47:f4:94:cf:4b:41:35: + d0:90:45:c4:43:c3:e3:b2:44:47:ba:46:4d:f6:2a: + b9:77:e0:fa:f5:74:d2:9e:11:1d:2d:ba:58:4c:db: + 04:45:b2:ac:34:6b:cb:cd:b3:5d:c8:df:40:47:e1: + a5:c0:af:96:c6:80:b0:85:da:d5:db:3f:72:a8:7a: + 84:7f:1a:44:12:59:5d:78:2a:b8:53:c6:12:83:dd: + 01:09:f2:01:e3:a0:76:0a:af:82:1d:70:56:1c:4d: + a9:1b:23:d1:37:4c:7b:10:8d:04:d4:30:9f:d7:ca: + 0c:da:ec:e3:ab:ea:93:4f:29:8d:74:b5:87:e9:ed: + 68:72:c9:8e:dc:07:f0:bf:ac:79:d4:63:47:f9:f0: + 86:65:45:d3:18:81:c2:c2:52:5e:f0:cd:4c:52:54: + 43:32:af:d0:2b:cb:c5:d9:66:5c:6e:3b:ad:bd:93: + 6b:9d:00:81:b0:22:2d:93:56:af:94:2b:49:0d:3d: + 9c:86:89:c1:87:31:e1:21:a0:d7:37:60:33:01:ff: + 36:1e:a8:ca:f8:2a:bc:88:fa:69 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 5B:93:42:58:B0:B4:18:CC:41:4C:15:EB:42:33:66:77:4C:71:2F:42 + C8:06:99:B7:E8:8F:EC:4F:3D:5C:89:6A:06:F5:77:2E:B0:E0:6A:9E X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,64 +103,64 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 5f:d8:9b:dc:22:55:80:47:e1:9b:04:3e:46:53:9b:e5:a7:4a: - 8f:eb:53:01:39:d5:04:f6:cf:dc:48:84:8a:a9:c3:a5:35:22: - 2f:ab:74:77:ec:a6:fd:b1:e6:e6:74:82:38:54:0b:27:36:e6: - ec:3d:fe:92:1a:b2:7a:35:0d:a3:e5:7c:ff:e5:5b:1a:28:4b: - 29:1f:99:1b:3e:11:e9:e2:e0:d7:da:06:4f:e3:7b:8c:ad:30: - f4:39:24:e8:ad:2a:0e:71:74:ab:ed:62:e9:9f:85:7e:6a:b0: - bb:53:b4:d7:6b:b8:da:54:15:5c:9a:41:cf:61:f1:ab:67:d6: - 27:5c:0c:a3:d7:41:e7:27:3e:58:89:d6:1f:3f:2a:52:cc:13: - 0b:4b:e6:d6:ba:a0:c7:fd:e3:17:a4:b8:da:cc:cb:88:70:21: - 3b:70:df:09:40:6c:e7:02:81:08:80:b0:36:77:fb:44:c5:cf: - bf:19:54:7c:d1:4e:1f:a2:44:9e:d8:56:0e:bf:4b:0b:e0:84: - 6f:bc:f6:c6:7f:35:7a:17:ca:83:b3:82:c6:4e:d3:f3:d8:30: - 05:fd:6d:3c:8a:ab:63:55:6f:c5:18:ba:66:fe:e2:35:04:2b: - ae:76:34:f0:56:18:e8:54:db:83:b2:1b:93:0a:25:81:81:f0: - 25:ca:0a:95:be:8e:2f:05:3f:6c:e7:de:d1:7c:b8:a3:71:7c: - 6f:8a:05:c3:69:eb:6f:e6:76:8c:11:e1:59:0b:12:53:07:42: - 84:e8:89:ee:ab:7d:28:81:48:e8:79:d5:cf:a2:05:a4:fd:72: - 2c:7d:b4:1c:08:90:4e:0d:10:05:d1:9a:c0:69:4c:0a:14:39: - 17:fb:4d:5b:f6:42:bb:46:27:23:0f:5e:57:5b:b8:ae:9b:a3: - 0e:23:59:41:63:41:a4:f1:69:df:b3:a3:5c:10:d5:63:30:74: - a8:3c:0c:8e:1c:6b:10:e1:13:27:02:26:9b:fd:88:93:7e:91: - 9c:f9:c2:07:27:a4 + 40:d1:6d:8e:a2:0b:91:4b:a8:c4:08:d0:f3:f9:8b:d0:a3:0b: + dc:00:22:8c:f1:2e:2b:e5:e6:b4:6e:ce:9d:cf:59:32:66:6c: + bb:0e:3b:1d:9c:05:d2:eb:a6:29:f8:74:4f:dc:83:3b:32:5a: + 2c:67:86:61:25:bc:bd:19:eb:20:c6:30:69:0e:4c:b2:e3:18: + ca:9e:fe:40:bc:1c:ad:8b:03:f5:04:be:90:ce:27:27:2f:83: + 14:57:8d:4f:a0:db:46:ce:e0:7d:e2:cf:7d:ea:0c:fd:8d:00: + 27:0a:db:0d:5f:e7:1e:52:25:1f:64:b9:30:5f:07:1a:10:a3: + 69:35:0e:dc:f8:23:f7:34:07:ce:c8:92:94:39:4d:d5:c3:ab: + 33:aa:f9:67:be:66:18:ac:67:14:5f:93:5f:68:48:04:ed:1e: + c9:74:28:b2:47:34:49:11:e4:7b:38:32:e5:dc:40:13:b4:69: + 75:39:43:db:7c:4a:f0:2b:94:cd:01:ba:4d:9b:9e:68:b3:ee: + 03:9e:7f:9c:0c:cf:9c:5c:cb:d4:33:d5:f0:e3:21:54:9a:13: + 6f:eb:1a:0f:f3:8b:e8:ef:eb:34:ba:09:77:39:2a:8a:4b:e1: + 7e:9f:b5:05:be:95:b6:92:5d:4c:35:47:38:64:38:5e:27:b8: + f9:34:94:2f:57:16:b0:f5:6a:21:3f:09:34:b9:dd:f8:d1:47: + 2c:c7:5e:7f:63:49:f4:5b:f4:d9:ea:66:fc:aa:64:27:f0:72: + d7:94:6f:86:0f:e7:3b:b3:d4:d9:30:67:b8:a2:c3:f7:4d:07: + 44:b3:70:67:dd:b1:21:ac:7c:2a:04:7b:2c:1d:df:0b:82:a9: + fb:df:88:72:47:1c:f5:5d:a3:f7:52:22:2d:ea:f4:2a:45:4f: + 9b:9d:63:95:59:f3:79:05:2b:f1:5b:3b:62:71:69:90:30:d7: + 7a:b2:c8:ec:68:e5:94:bb:97:00:d0:95:a7:fd:04:6c:f7:8b: + 28:c1:96:9b:6a:94 -----BEGIN CERTIFICATE----- -MIIGvTCCBSWgAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIGvzCCBSegAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk -bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCp6zt40WB3K7yj -BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 -nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd -1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc -LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO -jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV -WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj -nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx -MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAaOCAo4w -ggKKMIHhBgNVHREEgdkwgdaCB2lkbnNhbnOCH3huLS1rbmlnLTVxYS5pZG4ucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2dzc2NoZW4tbGNiMHcuaWRuYTIwMDMucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2djaGVuLWI0YTNkdW4uaWRuYTIwMDgucHl0 -aG9udGVzdC5uZXSCJHhuLS1ueGFzbXE2Yi5pZG5hMjAwMy5weXRob250ZXN0Lm5l -dIIkeG4tLW54YXNtbTFjLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0MA4GA1UdDwEB -/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/ -BAIwADAdBgNVHQ4EFgQUW5NCWLC0GMxBTBXrQjNmd0xxL0IwfQYDVR0jBHYwdIAU -8+yUjvKOMMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK -DB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNh -LXNlcnZlcoIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKG -MGh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNl -cjA1BggrBgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2Evb2NzcC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250 -ZXN0Lm5ldC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGB -AF/Ym9wiVYBH4ZsEPkZTm+WnSo/rUwE51QT2z9xIhIqpw6U1Ii+rdHfspv2x5uZ0 -gjhUCyc25uw9/pIasno1DaPlfP/lWxooSykfmRs+Eeni4NfaBk/je4ytMPQ5JOit -Kg5xdKvtYumfhX5qsLtTtNdruNpUFVyaQc9h8atn1idcDKPXQecnPliJ1h8/KlLM -EwtL5ta6oMf94xekuNrMy4hwITtw3wlAbOcCgQiAsDZ3+0TFz78ZVHzRTh+iRJ7Y -Vg6/SwvghG+89sZ/NXoXyoOzgsZO0/PYMAX9bTyKq2NVb8UYumb+4jUEK652NPBW -GOhU24OyG5MKJYGB8CXKCpW+ji8FP2zn3tF8uKNxfG+KBcNp62/mdowR4VkLElMH -QoToie6rfSiBSOh51c+iBaT9cix9tBwIkE4NEAXRmsBpTAoUORf7TVv2QrtGJyMP -XldbuK6bow4jWUFjQaTxad+zo1wQ1WMwdKg8DI4caxDhEycCJpv9iJN+kZz5wgcn -pA== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +aWRuc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAN4YbKyo2qzP +rj7Gw34JRkN8KA8+HNQrSUNSWQSSnRl4V5Cbf6mjZjzaC8khVqU7kKGJeM/FKAnC +i2hId7hBnx9YCWRfV3X0DzGON63HUrd2pmy2p7eY8/sdC98mVGSYC1WEVpSPX5JT +H0qzgcHguTaGopSirdOVEyl5J8lNdodOFU/spk9YHN8zul2kZifOcGnGB/KY2ulN +xc0KN5HqVSWP9MTEeJguPgTRo4NH9JTPS0E10JBFxEPD47JER7pGTfYquXfg+vV0 +0p4RHS26WEzbBEWyrDRry82zXcjfQEfhpcCvlsaAsIXa1ds/cqh6hH8aRBJZXXgq +uFPGEoPdAQnyAeOgdgqvgh1wVhxNqRsj0TdMexCNBNQwn9fKDNrs46vqk08pjXS1 +h+ntaHLJjtwH8L+sedRjR/nwhmVF0xiBwsJSXvDNTFJUQzKv0CvLxdlmXG47rb2T +a50AgbAiLZNWr5QrSQ09nIaJwYcx4SGg1zdgMwH/Nh6oyvgqvIj6aQIDAQABo4IC +jjCCAoowgeEGA1UdEQSB2TCB1oIHaWRuc2Fuc4IfeG4tLWtuaWctNXFhLmlkbi5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ3NzY2hlbi1sY2Iwdy5pZG5hMjAwMy5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ2NoZW4tYjRhM2R1bi5pZG5hMjAwOC5w +eXRob250ZXN0Lm5ldIIkeG4tLW54YXNtcTZiLmlkbmEyMDAzLnB5dGhvbnRlc3Qu +bmV0giR4bi0tbnhhc21tMWMuaWRuYTIwMDgucHl0aG9udGVzdC5uZXQwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB +Af8EAjAAMB0GA1UdDgQWBBTIBpm36I/sTz1ciWoG9XcusOBqnjB9BgNVHSMEdjB0 +gBTACisoQ95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNV +BAoMHVB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXIt +Y2Etc2VydmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcw +AoYwaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQu +Y2VyMDUGCCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9vY3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhv +bnRlc3QubmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOC +AYEAQNFtjqILkUuoxAjQ8/mL0KML3AAijPEuK+XmtG7Onc9ZMmZsuw47HZwF0uum +Kfh0T9yDOzJaLGeGYSW8vRnrIMYwaQ5MsuMYyp7+QLwcrYsD9QS+kM4nJy+DFFeN +T6DbRs7gfeLPfeoM/Y0AJwrbDV/nHlIlH2S5MF8HGhCjaTUO3Pgj9zQHzsiSlDlN +1cOrM6r5Z75mGKxnFF+TX2hIBO0eyXQoskc0SRHkezgy5dxAE7RpdTlD23xK8CuU +zQG6TZueaLPuA55/nAzPnFzL1DPV8OMhVJoTb+saD/OL6O/rNLoJdzkqikvhfp+1 +Bb6VtpJdTDVHOGQ4Xie4+TSUL1cWsPVqIT8JNLnd+NFHLMdef2NJ9Fv02epm/Kpk +J/By15Rvhg/nO7PU2TBnuKLD900HRLNwZ92xIax8KgR7LB3fC4Kp+9+Ickcc9V2j +91IiLer0KkVPm51jlVnzeQUr8Vs7YnFpkDDXerLI7GjllLuXANCVp/0EbPeLKMGW +m2qU -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem index 187021b8eeb9fa..1739a3525fee2b 100644 --- a/Lib/test/certdata/keycert.passwd.pem +++ b/Lib/test/certdata/keycert.passwd.pem @@ -1,69 +1,69 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIc17oH9riZswCAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDwi0Mkj59S0hplpnDSNHwPBIIH -EFGdZuO4Cwzg0bspLhE1UpBN5cBq1rKbf4PyVtCczIqJt3KjO3H5I4KdQd9zihkN -A1qzMiqVZOnQZw1eWFXMdyWuCgvNe1S/PRLWY3iZfnuZ9gZXQvyMEHy4JU7pe2Ib -GNm9mzadzJtGv0YZ05Kkza20zRlOxC/cgaNUV6TPeTSwW9CR2bylxw0lTFKBph+o -uFGcAzhqQuw9vsURYJf1f1iE7bQsnWU2kKmb9cx6kaUXiGJpkUMUraBL/rShoHa0 -eet6saiFnK3XGMCIK0mhS9s92CIQV5H9oQQPo/7s6MOoUHjC/gFoWBXoIDOcN9aR -ngybosCLtofY2m14WcHXvu4NJnfnKStx73K3dy3ZLr2iyjnsqGD1OhqGEWOVG/ho -QiZEhZ+9sOnqWI2OuMhMoQJNvrLj7AY4QbdkahdjNvLjDAQSuMI2uSUDFDNfkQdy -hqF/iiEM28PmSHCapgCpzR4+VfEfXBoyBCqs973asa9qhrorfnBVxXnvsqmKNLGH -dymtEPei9scpoftE5T9TPqQj46446bXk23Xpg8QIFa8InQC2Y+yZqqlqvzCAbN6S -Qcq1DcTSAMnbmBXVu9hPmJYIYOlBMHL8JGbsGrkVOhLiiIou4w3G+DyAvIwPj6j9 -BHLqa7HgUnUEC+zL4azVHOSMqmDsOiF3w9fkBWNSkOyNoZpe+gBjbxq7sp+GjAJv -1CemRC3LSoNzLcjRG2IEGs1jlEHSSfijvwlE4lEy3JVc+QK8BOkKXXDVhY1SQHcS -pniEnj95RFVmAujdFDBoUgySyxK/y6Ju/tHPpSTG9VMNbJTKTdBWAVWWHVJzBFhR -0Ug62VrBK7fmfUdH1b37aIxqsPND2De6WLm0RX+7r3XPDJ7hm+baKCchI5CvnG19 -ky8InhMbU4qV+9LceMETmNKKDkhKl4Zx/Y3nab7DG9s/RZfrTdCHojc9Va/t0Ykp -qlVrvdj/893CdI78SW3VjWBJGWfKMyT16hBMY3TPz6ulbFXk6Pul/KcLLWslghS+ -GKZjyBe96UwfH4C7WjuIB+zo+De3Wr8xOCdJR5zwEutBMM+L/Wul8B6wIEGS71kB -TN/CAoeIgHLQFbcw4YE80dllTnSEsqF+ahVTTcCt3iLUaOgeTUxteMbXY9+nekSX -x8aUcvkMhbU9omdEowFr5/HIMKXo4UXat4fIGgh2pG8v8fA46hZXkhWUh/PhbnQw -StXzn4fA13erqVI679kHMmOIQebv4oqdcwkImrH5fEsACNjQbkYZF5fD4z+1GHkA -e2eGqejVT+OV14I8qfx9oqs2f8aqijH8fYLU0TymE7p53DYZy4WvDwk22I4rMzoQ -sGkOZwfKUYpdBI2t6tEf1ROBjoNG0E2Onq+5iooibN08rKXKAQMWsK+2vNHNHwBW -49vRheQNnRqSuLY+b7QAjA0KuRWo9YptCbnXyF/Aw64jMfAGjggDLoaZfALGZk3n -P+ZoL9xc7rYRpIca44BeYI6AhHFcWWIOX7Sm69FvmyHlfsgTAXVgY1lQPuGy68Au -PHSkgUyydDtkrfb2W2gJuqD/+h+9X2z+o/+nETYPCZm3sH5xvTY/DTcTx9kTpXxx -YQBaFTt12eVX7wZVr5K3u9M371rg+SeXC2SzL4T6APHD52cxbA1jgM0JFh3KJTuk -fADxIzM1NdzYQ45J6i2w+/Fh4VPnXZ0oiUSwE094XTBlvhI6zHgar2Q0Qx1P51vB -odd9XzyDLULuIzei0DYjTIg0KhE+wAGq1I5qtiMhmy5TdCKKNA9WGb1Pq38zpyjU -wGmztzSzCEjfLyhChaUObVRRxEfD5ioxKer/fczOhKQe8FXmGy5u/04tVmmEyNOO -JkkDtZy+UbKuJ257QnY72wPjgtHNy+S4Iv7zHUbNJNhxk+xBlRcmRNWCEM20LBSO -Tj4S9gyan+gH2+WFxy8FaENUhM+vHFEeJcjQIBFBeWICmSmdkh/r0YK1UVJ9NLfR -l0HiKm3lKg+kNCexTAPLMt2rGZ4PAKVnhVaxtuHMYYDpl2GYmyH73B9BfcPdA/Zx -GUBmd9hwcLz9SuUg+fjHcogZRRRlcZlKhw3zUCsqHSCQXZCQm7mBlG/5C/7cM7wQ -IRtsNospLStOg51gv21ClQ+uWx30XEcwmnIfVoLl1vMaguuf1u5u3dWBD/UgmqiP -1Ym8jv0BF/AS+u/CtUpwe7ZWxFT0vbyi10xxIF7O07fwFa+5dME3ycZwcyiE95K1 -ftcHlGOIhuVBMSNZXC4I9LM+7IWy+hanUcK+v5RvwBDSJV3fnAOdfrka1L/HyEEb -x/FYKEiU/TAjXDw2NtZ2itpADTSG5KbdJSwPr01Ak7aE+QYe7TIKJhBDZXGQlqq8 -1wv77zyv7V5Xq2cxSEKgSqzB9fhYZCASe8+HWlV2T+Sd +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQEKGM7Z40p3TpyCvl +LTPLsQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEJAwQy7HESbbXMoF +KXadaz8EggcQBlxqFXmWQ3YlUuWTTc+dUcmikIUmI8gUSB2doBxQWLkTzaNuQ/hj +uV/eU7VkNFswxrU1j+EEkoCAtClWrQ2aKTyQSM4YfhgN1vQJdOD8DypHtd0TaRS3 +VCSkJODqcWaCEc/Ypgb1KnB6UWhit1waPq4NVCZdiSF/ueRK9/iMlIRaejg61Qn+ +LVLcegWw47zTtCZKuZqFAO3PcvfC0cresGc6hrQrb9tjUyI9qCR6MeZSrHUJQ/Pb +T9e7OREQxCLxWEUb0/tMJH5k9HoG7waU5lELOyjGKXsflEIs+uBaWIssWp69KGLi +yas7BHZc8kxJNtigFQlmECZjoKvRxJWN4fOknxI8T+ttBovOtsIlVvocBfmH+Z2m +7a5VMBxD8Phk6zU4Pk0L2S6EIUCIYGS+kgHibgxZ58/QcoDZBrKOaJ/Md+Lhk2Rn +2JCuPe8CBTkB3V+xAJdaz7JAcgxz4TqwXFXA249lfYb8qY9XXcr67O83Cxwb95lu +skmxnhAKOwKIrhS1nKiefuwN5qOiA+nuPbJFadCOO5cdOSriG3s80ugDDvlPN8k6 +4b7XLulB38R2PdH/OHuF05QGQ3kOUbJWatMig+/09LNo9FiMaCczdIXQ6LGxcHc1 +G1Y3BqK17z3yULqHvxU4tIbj5Elt+X1mfKIfifmXBUujbz5rR1pBTfpnk+DFGcHZ +B/cXWFP0tDJAE7mvna3HimNDQMP58NbhZwFtFdU3H9R5n4R5MsYQ6+a+amQ0pZ1Y +XAaQ/iEg0WY8CEFO8zEvB9R05zMm6vycNtN8CbfBq9CJ0OaR1ymMW8pRVagsLDuh +D0T5ZtWZctE2+ImlwlpGDs91CX3zDxvW/3Bwf40PF4x7LJMt/tFzqQEovokvzhUw +0jX/kf42QhvydnDoxqczdVZOfbjHELA21U+JAeAAj9jhcCEYd50c2BVt4jhST6dP +pzNAqxc1RGwTU3K3nZkWKt4hMXPDGjb0pVqiOuPz/718Nsd6ck+Ko90gFX13mPLC +6FLPGjNUmE6f3TKjbRrQz3+IWMncCyo/JSWA85rldxD0XAem64h+2XppFWJsxJD5 +nH2prLckQUWBLljWNIQQyVuAWo0TVq3NCXDFGaP6GHNrZo9zZxDhxcp6v2Me3Mir +few9aQNb59Q1/A0qwiCf26Oe8JYTD14UCez41imxD+w721SO8jzLR80+ReemdwzE +B9fXh3T/lrCIDNB+F+GWA9wQUz8mzArKbhqQhl5gk39dnLahc4aN7GrkwL4hjhKW +O3X5bBdhhRlcub9SbXFC+tx/w6G/0roGMSvD7h8G/mw3QcUUwwDvOuI7qAqQ8FC+ +xcGWeHUiTGxp2t2gz8DSaeL2TbBMbHe+tRBmKlmmIrAq3AEllWzyze9ta5ZYt46P +njZzS+vbbA0mfcFAId7tQOjQ2/ygZePs5hg91revLzNMDRJ1IeBHnrVUuDA5D1HR +iFW+hhj1Lx1s0XAJBJcDos8xy/LZlRDLxJSHrMktZOELTYh7UZw4S2PNkOPms4kP +V4G6D/0wZh0t+T9zpH57ivLdyBQ2fsFwx3X3JfQkYmoFlwCz+uUb368TunQ/yZZY +fVh9Q9ndk6y6aor0wkORsBefVv3eN+2BDt6ZUpYu3Liim+W4C/SiKVfQ+5yDb/qe +KmX7kcmr/f13BWvEOUN4fzCMiw/N0aAgEejZQTwMgivvGSMcNaBiLfdvM6hmy7NE +uvgpLtK5v+YRcOLwiaVH7UL6Q61CJ+pcDNRvbJT/iVwidYUO16zTVR6NrtPhM62a +Ziq3Q77HD4D2KCRPEo3Xivc5RwITHeDgKHwpLRB00jiZ3iR0Wm5lI5VbyWLnmQ4x +CDPAZetJZGBkbhTmUVbL0m3HNLHfF9g+LWA23L9mRNCKRemL0c0UJRRJPaNDIx8p +8h27al4xI8nFav0TZLxmSw7mqnzyE7YAoe4EsmedOSXktHWN06dnczHu8AWrgNjY +Izv9XUmy9V5vsRG/lg09FyG/eZSgfs9rdmL/qeFXbbl2J33YTqrt6O3ZaEsgg42r +piZztung8Xro3VlFSAXZGrl7Z0AWwzaOvwreCxbAG1WtBmGgLrs98z627XgfBFYA +BPvJwn9f2GZzixiN6M8c2M5XueDE/Vpn4A/GKLJ8RXXxKtbGRXFfdF/M6mEgcpa1 +9pxAhzNTPaOp+SFbm/cFa43HZYZgDg1D9zth2ZII0ZITd5OpEaSNOsrOrhqXeVQU +isBpybgPqVQ50xUuRUyoHYAZClVe6+PBYVbvlXBlTWhSPY3leUm4AYt+fjZ4VFlb +p4I2KRmd68zXXwl2spkEsC96e9k28kHta55dO32gSwaXUUHRKnFmKB24jsleF6n3 +BOKxzsO9jr6vi0jUqZXvaCnmYWqS84wZH/1S1pgotaWdMQ/KT27xwHk= -----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC -WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG -9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF -vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq -tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 -oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O -HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz -fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd -njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi -wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta -As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs -aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL -BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo -xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH -TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 -KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F -/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ -qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm -MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH -5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo -EGD2IF7VDQ== +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem index a30d15ca4d61a6..43ad52c2af507b 100644 --- a/Lib/test/certdata/keycert.pem +++ b/Lib/test/certdata/keycert.pem @@ -1,67 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 -qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M -V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz -hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td -ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID -eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb -SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV -uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ -HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO -M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP -MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ -tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP -qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq -D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph -7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq -U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG -ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC -gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ -u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG -XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s -5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo -dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH -Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 -UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr -r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 -XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 -1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx -tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz -og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX -v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z -SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 -uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 -+yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE -5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV -AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o -VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F -G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R -kNdcwhtZZ3dpChUdf+w/LIc= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC -WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG -9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF -vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq -tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 -oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O -HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz -fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd -njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi -wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta -As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs -aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL -BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo -xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH -TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 -KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F -/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ -qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm -MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH -5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo -EGD2IF7VDQ== +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem.reference b/Lib/test/certdata/keycert.pem.reference new file mode 100644 index 00000000000000..5528b98fff9377 --- /dev/null +++ b/Lib/test/certdata/keycert.pem.reference @@ -0,0 +1,13 @@ +{'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 29 11:51:12 2408 GMT', + 'notBefore': 'Oct 8 11:51:12 2024 GMT', + 'serialNumber': '4F1870457EB3050737FA5DE298392511BA9C0E92', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem index c7c4aa74583c8a..63e2181ed83e78 100644 --- a/Lib/test/certdata/keycert2.pem +++ b/Lib/test/certdata/keycert2.pem @@ -1,67 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCyAUXjczgUEn7m -mOwDMi/++wDRxqJAJ2f7F9ADxTuOm+EtdpfYr4mBn8Uz9e3I+ZheG5y3QZ1ddBYA -9YTfcUL0on8UXLOOBVZCetxsQXoSAuDMPV0IXeEgtZZDXe7STqKSQeYk7Cz+VtHe -lZ8j7oOOcx5sJgpbaD+OGJnPoAdB8l8nQfxqAG45sW4P6gfLKoJLviKctDe5pvgi -JC8tvytg/IhESKeefLZ4ix2dNjj2GNUaL+khU6UEuM1kJHcPVjPoYc+y8fop/qhQ -0ithBhO2OvJ+YmOFdCE67SyCwU3p8zJpN+XkwbHttgmNg4OSs7H6V7E52/CsTNTA -SthBHXtxqaM+vjbGARrz2Fpc/n+LwRt7MGIR0gVtntTgnP0HoeHskhAIeDtaPrZ6 -zHdl3aDwgAecVebTEBT5YPboz+X1lWdOrRD2JW3bqXSRIN3E4qz5IMuNx3VvhpSR -eFZzR6QIbxQqzO/Vp93Ivy8hPZ6WMgfSYWs7CGtu4NP79PJfdMsCAwEAAQKCAYAc -e3yp2NlbyNvaXRTCrCim5ZXrexuiJUwLjvNfbxNJDeM5iZThfLEFd0GwP0U1l86M -HGH2pr6d4gHVVHPW5wIeL9Qit3SZoHv9djhH8DAuqpw6wgTdXlw0BipNjD23FBMK -URYYyVuntM+vDITi1Hrjc8Ml7e5RUvx8aa5O3R3cLQKRvwq7EWeRvrTMQhfOJ/ai -VQGnzmRuRevFVsHf0YuI4M+TEYcUooL2BdiOu8rggfezUYA9r2sjtshSok0UvKeb -79pNzWmg9EWVeFk+A0HQpyLq+3EVyB5UZ3CZRkT0XhEm1B7mpKrtcGMjaumNAam7 -jkhidGdhT/PV9BB1TttcqwTf+JH9P9sSpY9ZTA1LkkeWe9Rwqpxjjssqxxl6Xnds -+wUfuovVvHuBinsO+5PLE5UQByn21WcIBNnPCAMvALy/68T7z8+ATJ+I2CqBXuM2 -z5663hNrvdCu93PpK4j/k/1l3NTrthaorbnPhkmNYHQkBicpAfFQywrv6enD+30C -gcEA7Vlw76og4oxI7SSD6gTfo85OqTLp2CUZhNNxzYiUOOssRnGHBKsGZ8p0OhLN -vk9/SgbeHL5jsmnyd8ZuYWmsOQHRWgg1zO3S25iuq+VAo4cL/7IynoJ0RP5xP0Pw -QD+xJLZQp0XuLUtXnlc6dM5Hg7tOTthOP9UxA1i57lzpYfkRnKmSeWi+4IDJymOt -WoWnEK7Yr7qSg6aScLWRyIvAPVmKF9LToSFaTq0eOD0GIwAQxqNbIwN3U0UJ5Ruc -KRBVAoHBAL/+DNGqnEzhhWS6zqZp2eH90YR+b3R4yOK4PROm2AVA3h1GhIAiWX68 -PvKYZK9dZ9EdAswlFf9PVQYIXUraR3az0UiIywnNMri+kO1ZxwofGvljrOfRRLg0 -B46wuHi6dVgTWzjTl503G9+FpAYNHv184xsr1tne0pf2TKEnN7oyQciCV8qtr8vV -HrL46uaD0w1fcXIXbO3F/7ErLsvsgLzKfxR5BeQo6Fq0GmzD+lCmzVNirtfl2CZj -2ukROXUQnwKBwQDR1CqFlm/wGHk4PPnp31ke5XqhFoOpNFM1HAEV5VK0ZyQDOsZU -mCXXiCHsXUdKodk0RpIB80cMKaHTxbc7o0JAO50q7OszOmUZAggZq1jTuMYgzRb3 -DvlfLVpMxfEVu7kNbagr2STRIjRZpV/md569lM+L4Kp8wCrOfJgTZExm8txhFYCK -mNF2hCThKfHNfy7NDuY9pMF2ZcI8pig1lWbkVc5BdX7miifeOinnKfvM4XfzQ+OE -NsI8+WHgC+KoYukCgcBwrOpdCmHchOZCbZfl9m1Wwh16QrGqi1BqLnI53EsfGijA -yaftgzs+s7/FpEZC3PCWuw3vPTyhr69YcQQ/b8dNFM8YYJ+4SuMfpUds5Kl5eTPd -dO/+xMQtzus4hOJeiB9h50o8GYH7VGJZVhcjLgQoBGlMgvf+uVSitnvWgCumbORK -hqR7YF+xoov3wToquubcDE2KBdF54h/jnFJEf7I2GilmnHgmpRNoWBbCCmoXdy09 -aMbwEgY+0Y+iBOfRmkUCgcEAoHJLw7VnZQGQQL4l3lnoGU9o06RPkNbpda9G/Ptz -v+K7DXmHiLFVDszvZaPVreohuc0tKdrT0cZpZ21h0GQD4B6JX66R/y6CCAr0QpdA -pFZO9sc5ky6RJ4xZCoCsNJzORNUb36cagEzBWExb7Jz2v6gNa044K5bs3CVv5h15 -rJtTxZNn/gcnIk+gt//67WUnKLS4PR5PVCCqYhSbhFwx/OvVTJmflIBUinAclf2Q -M4mhHOfwBicqYzzEYbOE9Vk9 +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCtFmmyQ+HjjeBX +UE61fTenp2C9cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4v +ZADh/eIFOONvPebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5b +yhhpcOH1AmPCsiLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKTh +J+X8+PCY+FRObAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpO +G35/bW9jsQRpbeGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJ +izTuFV91pGmsXSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyU +Vr+odU0t7+YExBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp ++5cGWLOkcs8Xt0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAQKCAYAU +x+HCxFb6LLjS4tJrBHCf0VRSX9rf/7Ri2oYRXRUN1B3cTrZFeZyLdJyKggNiSy3E +GcF4lrxl2Xgaz8/cp3rxE5EYHv93hCQPZUbu4r2kZCLJsRPcGn3D/dyEKhCGyAi/ +fgsTkxyxnyK/9HXqhYyrZNu4rrh0SH63GKJ4GZsIFhZME95b0f6/e/3YJpSJFxKK +LF+xd/rjoWNOS62s1JqA/czpaXDqnJ0frNX5aQixLHnWUy/5qgRGNnY3/LOB0MuU +EjFizRSJp6gQQ64dIFgiMyUMi+Z6fw6xkTpN5EbmgFCwmaja1/hp9UZ+atlVpJsO +SB+OhF2wxB+bX2phGHJ9DD4RY/di1YKPvmH0+LITq4JiJWwJLgoCm7krzfVxZFZN +lgZPvdNw/Yt/Y3bN31FMw8DwRCThnL+MH83E58iCWQq2zKS1RssBBwFacI6Wf1ed +442NPIH47oi5qRKUxYHze6kFssSs4TkbR++VYHXLbTLjqfePLP92ZFxdOstx5jEC +gcEA48mx39dwr1oSk4uR+q8aDZIjPT+9D0IxcdAmL968QgG98hF2Tx7s5alR4y+9 +2FueR5fSqRweJfaDn77j5DVNXLD5jBz73pdwIkfqJHLfcsN7NnzCz3mxU9WP2F7R +W8KD3C2MtzGwQwOI3GT20+wJaA7p5n5zKR7AC0RxhC3U4zwH5nSjtY70A3dWXkn3 +F0j+B6ak0P0oN9VQUuvk5zw0IZ32uxef7poiHvFZpl1DQSgX/edT1CIT1s0nmVFl +w1ojAoHBAMKGX+TobGZfZFj/I7oIVQ7023pi516fZx/R0Y0FTmZzIKsdt3wrPmar +3K5dZGasNd0wzUsxX+vSjMtI1oERuPd4Rs6YqQHpX892uJ7L41/oBcFCc+Fk7jBY +TiRg/8/gOTaQe1nDR52K/R0sqBatRqInNMBDcBsVguBC4dTqP8CQ0CnEjAOYsk1s +1nYPKcfM36BUXBmSlnvUPWMm8KCT9Kko9ylHVq+d+ziDebfYpy6fTvti2NM31wTM +1prasEUaqQKBwQDQX14+5MapMd1SaVelmW5cwbVIvzjEb4npkj6MhdVzEELg4IZ4 +hFKzGDvXdoHVHKJi3YiQuC8ADUyE4kt4JCZbx2zQdmcVTTT/tweCRi8PvbDFvEU9 +JBZKGU+X38zmgr66uFRD9MlH1EDrU9TTMdW9Af+HoV7ZW87Tv82T25UmNXEIqORl +HpsrXIx+fmzxOQ1glFmq8BpNUO5EnJPtz43kvqrIpSjhTNAvvBqFbEUsom+oDWgK +4w2A7nTt9J8BoD8CgcBkYKa3HmBhazQC4JV097u8nglrXAH8R9EVEFZLqMNOBnaD +FjCKeF4Y6PJVX4fhm1eoLfihpnbS37EbbRiTPavutzgCf7AmdmCkU6Ts/FT2NmpR +0ZKuakCm3cpk51DZ2eBsEZ41MZmQ6Bm4pkSOfxeFsSl9VM9SioUgaCLUlZQUMCXa +h7ugV3kajuETxrtOiJ+UwjNMVuIkP971fTCKDA8iAyuXN2K5+JGcFewHPFr4qeg9 +vEIarCPeLD1JZzOyVRECgcEAhqeXfmFezNKGvr7qCflJe92LF9MStgJ7yE0OfADy +B+RZKeOwoqOO3VFR1piAn/DzrC2K9q+Of61gw+KWQOgMnsigqZ2mFLGChRjWhb7S +3G0DGOb4+DD9RR6wlFPFXSwVxSWGKrqNhOJik/IzVWvYxOOFjVt1adXZ3GftnYsv +nZCsS94H4kMiXr6UkbkjjxnZ1WkE9DaQJU27Mw1dtwb1ECzO0rty9B9L6b80MYW/ +nMhJV6sanCUex4nAChHD+VR7 -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEjDCCAvSgAwIBAgIUQ2S3jJ5nve5k5956sgsrWY3vw9MwDQYJKoZIhvcNAQEL +MIIEjjCCAvagAwIBAgIUCP/QP57z62jCsjf6l0ft/yzDxOkwDQYJKoZIhvcNAQEL BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv -c3RuYW1lMB4XDTIzMTEyNTA0MjEzN1oXDTQzMDEyNDA0MjEzN1owYjELMAkGA1UE -BhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24g -U29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhvc3RuYW1lMIIBojAN -BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsgFF43M4FBJ+5pjsAzIv/vsA0cai -QCdn+xfQA8U7jpvhLXaX2K+JgZ/FM/XtyPmYXhuct0GdXXQWAPWE33FC9KJ/FFyz -jgVWQnrcbEF6EgLgzD1dCF3hILWWQ13u0k6ikkHmJOws/lbR3pWfI+6DjnMebCYK -W2g/jhiZz6AHQfJfJ0H8agBuObFuD+oHyyqCS74inLQ3uab4IiQvLb8rYPyIREin -nny2eIsdnTY49hjVGi/pIVOlBLjNZCR3D1Yz6GHPsvH6Kf6oUNIrYQYTtjryfmJj -hXQhOu0sgsFN6fMyaTfl5MGx7bYJjYODkrOx+lexOdvwrEzUwErYQR17camjPr42 -xgEa89haXP5/i8EbezBiEdIFbZ7U4Jz9B6Hh7JIQCHg7Wj62esx3Zd2g8IAHnFXm -0xAU+WD26M/l9ZVnTq0Q9iVt26l0kSDdxOKs+SDLjcd1b4aUkXhWc0ekCG8UKszv -1afdyL8vIT2eljIH0mFrOwhrbuDT+/TyX3TLAgMBAAGjOjA4MBcGA1UdEQQQMA6C -DGZha2Vob3N0bmFtZTAdBgNVHQ4EFgQU5wVOIuQD/Jxmam/97g91+igosWQwDQYJ -KoZIhvcNAQELBQADggGBAFv5gW5x4ET5NXEw6vILlOtwxwplEbU/x6eUVR/AXtEz -jtq9zIk2svX/JIzSLRQnjJmb/nCDCeNcFMkkgIiB64I3yMJT9n50fO4EhSGEaITZ -vYAw0/U6QXw+B1VS1ijNA44X2zvC+aw1q9W+0SKtvnu7l16TQ654ey0Qh9hOF1HS -AZQ46593T9gaZMeexz4CShoBZ80oFOJezfNhyT3FK6tzXNbkVoJDhlLvr/ep81GG -mABUGtKQYYMhuSSp0TDvf7jnXxtQcZI5lQOxZp0fnWUcK4gMVJqFVicwY8NiOhAG -6TlvXYP4COLAvGmqBB+xUhekIS0jVzaMyek+hKK0sT/OE+W/fR5V9YT5QlHFJCf5 -IUIfDCpBZrBpsOTwsUm8eL0krLiBjYf0HgH5oFBc7aF4w1kuUJjlsJ68bzO9mLEF -HXDaOWbe00+7BMMDnyuEyLN8KaAGiN8x0NQRX+nTAjCdPs6E0NftcXtznWBID6tA -j5m7qjsoGurj6TlDsBJb1A== +c3RuYW1lMCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBiMQswCQYD +VQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhv +biBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDDAxmYWtlaG9zdG5hbWUwggGi +MA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCtFmmyQ+HjjeBXUE61fTenp2C9 +cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4vZADh/eIFOONv +PebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5byhhpcOH1AmPC +siLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKThJ+X8+PCY+FRO +bAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpOG35/bW9jsQRp +beGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJizTuFV91pGms +XSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyUVr+odU0t7+YE +xBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp+5cGWLOkcs8X +t0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAaM6MDgwFwYDVR0RBBAw +DoIMZmFrZWhvc3RuYW1lMB0GA1UdDgQWBBSAf/Z+TUHMOcr020NGG4geSnlNhTAN +BgkqhkiG9w0BAQsFAAOCAYEAIia0ULQmC510p78HZbiOb2BV8q+0AY0l6Cn/FRIL +Voqy7uB5oEYdDisla8epmBTU35+JmfrRHbP6IuSqKdcGj8VqsjZljhntSrB7rw4O +IqjbnxnfEuREjY2w+WGLvvdtVGXCBgfRmIItORFKpoOvLzLIi1lXeDq8QL97K4nM +tPsZpzILCBCoXmhh0MweCCNe1HSD8q6EbSDjVoxraBl0YK/l1ID08Fi5fwdddRX7 +txfx5NtoWWbsLZY0GdBtQmRCifs7P9lyhjUd87bJGd8WCBE4IzLwJiHEyNPI4WHe +8jmvKD/zO7mKxT9/jF7IBqwTHgQQ1uaRGytiXjRlfPu4Ez8ASR6rP7Hlua7h0Ba7 +OcDPcgM8rzN4zcfsvZ+8Cd84HXFjHhV6ZdRPbBWFWd8TlDufrWNuY4SyOtkyfE5E +lWcoRnkBGBaooZNcd72ZwZVO0gAS31bgs8ju+k02GIDLkuZSRNPeFsRQbcMu1Mhc +cFteFsqniLhzpkWJVQQgeGlV -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem index 20d9bd14e9678f..097dc9f8e88091 100644 --- a/Lib/test/certdata/keycert3.pem +++ b/Lib/test/certdata/keycert3.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCgKihxC+2g7d7M -JfIUBfFWiuMwxg0WhdxPyGUzMAjexbEOHo0ojntxPdH9KYRwiKRKb9jnmzXp2CKT -hqBYJIetq/3LYZp4bvDJ/hVCL9e4jqu1l+wd9DkqhKZ69b6C1/d12JAKvC5TIT+/ -b7EglYU8KMNx2WO5KxIJeVpX68jn49YtUzg0hT0QiXj4eugbDk1L1f99xgvkHaVW -VQxi6MFNWHJq/xXUb8E/hd/Q3oadN1BXMWl9P46D4R+YGKQQdZFkwEJsbqijFvWW -bOoaz7TFxf8n0q616803aXLfaWikfEXLnznEvKo7vyEivtT/y14Nm+SiR3nS6E0y -Dt8gmeHdaHcrmQT+yQ6yNOYDCcfeYM+rBuvOUHPudjMy0k8K/0IPjDo0KActKPH0 -UVbyDBMKDdGQ2+LhRFLsGXHlD9b05PxhqTULe3LeK6KZ+iuGbWtwVLaL5S42WiCA -YXNShE1Ko0Q7wugAippXCf+aWP3Wx9ZTrsfiDBbIfnY5mlfdG90CAwEAAQKCAYAA -ogoE4FoxD5+YyPGa+KcKg4QAVlgI5cCIJC+aMy9lyfw4JRDDv0RnnynsSTS3ySJ1 -FNoTmD5vTSZd1ONfVc2fdxWKrzkQDsgu1C07VLsShKXTEuWg/K0ZKOsLg1scY0Qc -GB4BnNrGA1SgKg3WJiEfqr2S/pvxSGVK2krsHAdwOytGhJStSHWEUjbDLKEsMjNG -AHOBCL5VSXS00aM55NeWuanCGH36l/J4kMvgpHB9wJE1twFGuHCUvtgEHtzPH9fQ -plmI0QDREm6UE6Qh01lxmwx3Xc5ASBURmxs+bxpk94BPRpj8/eF2HPiJalrkJksj -Xk3QQ7k23v6XnmHKV3QqpjUgJTdbuMoTrVMu14cIH6FtXfwVhtthPnCI8rk5Lh8N -cqLC7HT+NE1JyygzuMToOHMmSJTQ8L6BTIaRCZjvGTPYaZfFgeMHvvhAJtP5zAcc -xQzyCyNBU8RdPGT8tJTyDUIRs20poqe7dKrPEIocKJX7tvNSI2QxkQ96Adxo1gEC -gcEAvI8m6QCDGgDWI8yTH9EvZQwq+tF8I+WMC+jbPuDwKg5ZKC7VjRO//9JzPY+c -TxmLnQu64OkECHbu7pswDBbtnPMbodF9inYEY5RkfufEjEMJGEdkoBJWnNx78EkV -bcffWik0wXwdt6jd1CAnjmS9qaPz0T1NV8m5rQQn5JUYXlC9eB2kOojZYLbZBl3g -xUSRbIqHC7h8HuyAU26EPiprHsIxrOpbxABFOdvo2optr50U7X10Eqb4mRQ4z22W -ojJdAoHBANlzJjjEgGVB9W50pJqkTw8wXiTUG8AmhqrVvqEttLPcWpK6QwRkRC+i -5N1iUObf/kOlun2gNfHF6gM68Ja9wb2eGvE5sApq9hPpyYF0LS3g8BbJ9GOs6NU9 -BfM1CkPrDCdc4kzlUpDibvc6Fc9raCqvrZRlKEukqQS8dumVdb74IaPsP6q8sZMz -jibOk0eUrbx2c5vEnd0W8zMeNCuCwO1oXbfenPp/GLX9ZRlolWS/3cQoZYOSQc9J -lFQYkxL3gQKBwQCy3Pwk9AZoqTh4dvtsqArUSImQqRygFIQXXAh1ifxneHrcYijS -jVSIwEHuuIamhe3oyBK6fG8F9IPLtUwLe8hkJDwm8Misiiy5pS77LrFD9+btr/Nk -4GBmpcOveDQqkflt1j6j9y9dY4MhUGsVaLx86fhDmGoAh2tpEtMgwsl91gsUoNGD -cQL6+he+MVkg510nX/Sgipy63M8R1Xj+W1CHueBTTXBE6ZjBPLiSbdOETXZnnaR4 -eQjCdOs64JKOQ0UCgcBZ4kFAYel48aTT/Z801QphCus/afX2nXY5E5Vy5oO1fTZr -RFcDb7bHwhu8bzFl3d0qdUz7NMhXoimzIB/nD5UQHlSgtenQxJnnbVIAEtfCCSL1 -KJG+yfCMhGb7O0d8/6HMe5aHlptkjFS2GOp/DLTIQEoN9yqK6gt7i7PTphY/1C2D -ptpCZzE32a2+2NEEW67dIlFzZ/ihNSVeUfPasHezKtricECPQw4h3BZ4RETMmoq+ -1LvxgPl3B8EqaeYRhwECgcEAjjp/0hu/ukQhiNeR5a9p1ECBFP8qFh6Cpo0Az/DT -1kX0qU8tnT3cYYhwbVGwLxn2HVRdLrbjMj/t88W/LM2IaQ162m7TvvBMxNmr058y -sW/LADp5YWWsY70EJ8AfaTmdQriqKsNiLLpNdgcm1bkwHJ1CNlvEpDs1OOI3cCGi -BEuUmeKxpRhwCaZeaR5tREmbD70My+BMDTDLfrXoKqzl4JrRua4jFTpHeZaFdkkh -gDq3K6+KpVREQFEhyOtIB2kk +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDuNnCqYWoguzKI +B3PwOsoRzI8wgNdxx3RiIJSItFISlAgB4XkYzPBpi5nFR8GCqq7IUQQ+iYwdZa3y +H6X11IdlrCx4gDdQFSFOegnG41oqv7GwBEFvq6xlowpoaGw+5/8/sJX/VU44rL9/ +fiMVTbehMhMS/z5nTSYSACV7sRXnwW4T6l2Od77FzPBjnaFFDNr5GIoiD4mNSJ35 +t05tliGcz0IyE5TGR6kIEFNIH5theHoJJ4JC4uLyWJweJpxDx10njWRwKN/tXm8F +nHQPMRAUl2wM07QZEI1yfzUr00/oG2dpPnPyNh+6p2ld8JS0g2/Pd2pQL8OtmM2c +DcN5E+qCEW1UVIJ5//ad8pUcoQwyRIBROJbakqv6FcVTdgTogoq5Vu+T6TVFLSIS +vVvpsV0MtT/H9/9TtCnqOwliE8s8kb3NjQT8e72VHNd43bVPE4DeQ+XlbMoKzV+G +OmSHSfzSOZB9kScSmGd5YD8783TbL8/7Nn1ERQdWqaixGI36+BECAwEAAQKCAYBV +Ubhevg9V89ZwdELpSxUu9NZgZ/VCck7UCplIsVUoBE8t5UULRfPhybdkuoOrulhp +tOLRR1ChAtcffohhmSJ5nwY6jqnBDCBmzD0OOEYGQ6xvv8Z0KcfQi2nh5WzHxy5b +8HJ5BmPC1tSr5FDKg5B+ssG0Lyl5tF8rWVQTjmSrIlO73Fhv+6GflPyQoVeutKEF +UO3Ar1H0AYtbcnUruPcHBBDQgMTrk6UOF1LM5U0wxwbmmnkEXeEtLeGUxv13JUMe +QTrSw0P+hn5uiDMwY5lI212ayorbwyNuzU0hGW7j98qa/S9MVSeRwv4bsbtBqwtT +ZVGL2TogjeJ+8+qDlv5tf3crD1pV0yu7uVUeIGrJZJdiiMM6N7/x3cIJtrLDaF1w +9kUqdQGWDWwRV3w0FaeXDVlUTJ3VFTZIinLo1Vmd3AXLn97eVy+68DWD6ELlMkfF +ro3bQbQu59DGswDF9+mMauhgvljnxY/wUnkyp8LGZt8w/RU/op8LcvPKFjQfX6UC +gcEA/RClguU/JTWQSF4Vjy5+DmrzSCXabrO7986lkOwPXEaXLsF2pGOU+sbxffXZ +PCQ4zU6aNYBMtNWLrVqBrPnI78XZQbmSG7zUDxWwmr1ttuSHGthdaZUVTOsFsnpE +R+1lj20G2T2ankzv0ICFz7BWyaYwyzDrABwuP189b3juuX6xGLlCO25/AnK+iTXQ +WF4ZURZiwDg4WpxvPsE2FFglN09TtDDCeQw/zv7Jh6jWtHZfivfv/wR8ILu72gde +LdinAoHBAPD5slojp2St6iWTaxcYYBc+4x+Rv7oyxeAyM/00iIAnf19oaAw3jki1 +yz6zeNnhaZuwa6ruvhJAESPc+pXkmoPpQ1yIRjKZc3jPVoaxYuSPgUEbHV6BQbjD +clIoGkC6ISsjanMq8Z3pK7e2bxY4mxGRQIFSFnTeK8m8nzq5dYisVtxxJFnV2fDG +NWDMdDYBh52vpkPjnm5OoOuilkEH2ygtZlT6bKAbnhi/FVjYCHk46inaYx8PakMu +LnvZI0OIhwKBwBGzYGBPeKM5o+Xr7sYdEmQfxvR88VJc8ADdS2dfm5NwvJJgpdPJ +w1nnIG0XDSLPxclWfiLP3o2ngiWV9wwKTKu4wwF94WJfStXjRn8MUOhCA9E04RPJ +gbvnlHZvZudBC6GElr4LOQ1phDypQLLOOsPQBAmyWj2fuvxjxQBPDSOcYPbBvog5 +qliZfgpK4U/NBShO0IlxZT+xQXa6PPYfVDsSKWCpKHEfEjeASshaXuowfW5S+U51 +GdmQSAtwCH5ccQKBwQCOLgW5gYfms2aPvSdWfR9VF9nSaqCBMCvoWDasky5mzucs +V+HsM2tUI09EM4h+pa02GyWruSmUgxCZ5GxFvJgedKc2FYG1oSysf0lCN69tw+4z +h9gQRpuMdGUjbF3xCuE/HqpUQWZGEamlv5JTvhpghx9ULibp1Zxob05Ty9E5TtYB +QxB7oN3yXkBoWLnIk6Z8t4KWU9rKosH3xfp5bDU2w3K5ePhWj3T8jOH/hZeaTqZ7 +A0uwq9u6v6jVkgxocEkCgcEAhyMPyiMqqq55qD6JlPY4zf4mWQJS4MnmWexH/5N1 +a6uogerwZnIjuH0/wNc2qzQSOwVJRqjlxRZbEkWST0JK/meXZ+e0nqaROWICutO6 +ciIf2wVjTzn1f85DHwPhUf3P2zIfpVFh+XuvSb32J+pzNwK40gMhirYMDSbsm/RQ +JrJHQwEX9BvI6IP5kaMOMlSwiG3Soo3MxeJWmozkz09+cs7BwMFsUKOdqruJHSaw +/Q06DQ9u9UyjmLyKv3IkN/cK -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:a0:2a:28:71:0b:ed:a0:ed:de:cc:25:f2:14:05: - f1:56:8a:e3:30:c6:0d:16:85:dc:4f:c8:65:33:30: - 08:de:c5:b1:0e:1e:8d:28:8e:7b:71:3d:d1:fd:29: - 84:70:88:a4:4a:6f:d8:e7:9b:35:e9:d8:22:93:86: - a0:58:24:87:ad:ab:fd:cb:61:9a:78:6e:f0:c9:fe: - 15:42:2f:d7:b8:8e:ab:b5:97:ec:1d:f4:39:2a:84: - a6:7a:f5:be:82:d7:f7:75:d8:90:0a:bc:2e:53:21: - 3f:bf:6f:b1:20:95:85:3c:28:c3:71:d9:63:b9:2b: - 12:09:79:5a:57:eb:c8:e7:e3:d6:2d:53:38:34:85: - 3d:10:89:78:f8:7a:e8:1b:0e:4d:4b:d5:ff:7d:c6: - 0b:e4:1d:a5:56:55:0c:62:e8:c1:4d:58:72:6a:ff: - 15:d4:6f:c1:3f:85:df:d0:de:86:9d:37:50:57:31: - 69:7d:3f:8e:83:e1:1f:98:18:a4:10:75:91:64:c0: - 42:6c:6e:a8:a3:16:f5:96:6c:ea:1a:cf:b4:c5:c5: - ff:27:d2:ae:b5:eb:cd:37:69:72:df:69:68:a4:7c: - 45:cb:9f:39:c4:bc:aa:3b:bf:21:22:be:d4:ff:cb: - 5e:0d:9b:e4:a2:47:79:d2:e8:4d:32:0e:df:20:99: - e1:dd:68:77:2b:99:04:fe:c9:0e:b2:34:e6:03:09: - c7:de:60:cf:ab:06:eb:ce:50:73:ee:76:33:32:d2: - 4f:0a:ff:42:0f:8c:3a:34:28:07:2d:28:f1:f4:51: - 56:f2:0c:13:0a:0d:d1:90:db:e2:e1:44:52:ec:19: - 71:e5:0f:d6:f4:e4:fc:61:a9:35:0b:7b:72:de:2b: - a2:99:fa:2b:86:6d:6b:70:54:b6:8b:e5:2e:36:5a: - 20:80:61:73:52:84:4d:4a:a3:44:3b:c2:e8:00:8a: - 9a:57:09:ff:9a:58:fd:d6:c7:d6:53:ae:c7:e2:0c: - 16:c8:7e:76:39:9a:57:dd:1b:dd + 00:ee:36:70:aa:61:6a:20:bb:32:88:07:73:f0:3a: + ca:11:cc:8f:30:80:d7:71:c7:74:62:20:94:88:b4: + 52:12:94:08:01:e1:79:18:cc:f0:69:8b:99:c5:47: + c1:82:aa:ae:c8:51:04:3e:89:8c:1d:65:ad:f2:1f: + a5:f5:d4:87:65:ac:2c:78:80:37:50:15:21:4e:7a: + 09:c6:e3:5a:2a:bf:b1:b0:04:41:6f:ab:ac:65:a3: + 0a:68:68:6c:3e:e7:ff:3f:b0:95:ff:55:4e:38:ac: + bf:7f:7e:23:15:4d:b7:a1:32:13:12:ff:3e:67:4d: + 26:12:00:25:7b:b1:15:e7:c1:6e:13:ea:5d:8e:77: + be:c5:cc:f0:63:9d:a1:45:0c:da:f9:18:8a:22:0f: + 89:8d:48:9d:f9:b7:4e:6d:96:21:9c:cf:42:32:13: + 94:c6:47:a9:08:10:53:48:1f:9b:61:78:7a:09:27: + 82:42:e2:e2:f2:58:9c:1e:26:9c:43:c7:5d:27:8d: + 64:70:28:df:ed:5e:6f:05:9c:74:0f:31:10:14:97: + 6c:0c:d3:b4:19:10:8d:72:7f:35:2b:d3:4f:e8:1b: + 67:69:3e:73:f2:36:1f:ba:a7:69:5d:f0:94:b4:83: + 6f:cf:77:6a:50:2f:c3:ad:98:cd:9c:0d:c3:79:13: + ea:82:11:6d:54:54:82:79:ff:f6:9d:f2:95:1c:a1: + 0c:32:44:80:51:38:96:da:92:ab:fa:15:c5:53:76: + 04:e8:82:8a:b9:56:ef:93:e9:35:45:2d:22:12:bd: + 5b:e9:b1:5d:0c:b5:3f:c7:f7:ff:53:b4:29:ea:3b: + 09:62:13:cb:3c:91:bd:cd:8d:04:fc:7b:bd:95:1c: + d7:78:dd:b5:4f:13:80:de:43:e5:e5:6c:ca:0a:cd: + 5f:86:3a:64:87:49:fc:d2:39:90:7d:91:27:12:98: + 67:79:60:3f:3b:f3:74:db:2f:cf:fb:36:7d:44:45: + 07:56:a9:a8:b1:18:8d:fa:f8:11 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 3F:B1:E9:4F:A0:BE:30:66:3E:0A:18:C8:0F:47:1A:4F:34:6A:0F:42 + 16:22:C9:AA:20:D4:F4:8E:ED:9C:A6:BB:23:F5:F2:29:44:ED:F2:1E X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,59 +103,59 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - ca:34:ba:c5:d0:cf:27:31:32:d6:0d:27:30:b8:db:17:df:b7: - 39:a7:bb:b1:3b:86:c4:31:fd:fb:ab:db:63:1a:cc:90:ab:b9: - 4e:ab:34:49:0c:5e:8c:3e:70:a3:a7:6b:2f:a7:9a:25:7b:01: - 5a:18:96:48:76:f8:36:78:74:fa:bc:7d:68:7f:e5:ca:a6:9d: - 7b:dc:72:bd:a3:25:51:17:68:e8:e9:d7:02:86:2c:7d:16:7c: - b5:dc:44:b2:0a:e3:f7:c9:33:a3:51:36:83:bc:d4:70:cd:84: - 91:9f:06:ba:2d:d2:05:0a:65:c3:d9:55:09:a8:b8:09:69:bb: - 93:86:c2:b7:c2:90:74:7c:bf:f0:5d:bc:0e:63:13:8c:eb:fa: - 0f:f1:fa:e5:12:70:4d:0c:eb:8c:2e:a2:42:42:00:04:0f:fc: - f9:1f:41:9c:63:78:f0:66:93:b2:8f:2e:e8:93:1c:50:cb:2d: - 7f:b6:ba:57:6f:52:62:d7:39:0b:09:82:ab:a6:53:4d:cc:05: - 3e:19:f0:d4:c0:ce:a9:ad:10:ce:b9:71:e4:8f:f2:5a:3c:65: - ba:dc:cb:e0:04:90:2b:a5:15:a6:7d:da:dc:a3:b5:b7:bc:a0: - de:30:4e:64:cb:17:0d:3a:a0:52:d2:67:3b:a2:3a:00:d5:39: - aa:61:75:52:9f:fe:9b:c0:e8:a0:69:af:a8:b3:a3:1d:0a:40: - 52:04:e7:3d:c0:00:96:5f:2b:33:06:0c:30:f6:d3:18:72:ee: - 38:d0:64:d3:00:86:37:ec:4f:e9:38:49:e6:01:ff:a2:9a:7c: - dc:6a:d3:cb:a8:ba:58:fb:c3:86:78:47:f1:06:a6:45:e7:53: - de:99:1d:81:e6:bc:63:74:46:7c:70:23:57:29:60:70:9a:cc: - 6f:00:8e:c2:bf:6a:73:7d:6e:b0:62:e6:dc:13:1a:b9:fe:0f: - c2:d1:06:a1:79:62:7f:b6:30:a9:03:d0:47:57:25:db:48:10: - d1:cf:fb:7d:ac:3d + 43:82:1f:18:b4:cb:7a:b7:4e:ac:c3:65:2e:b7:57:3a:f0:78: + b3:b2:20:3e:bd:7b:24:f7:e7:17:cc:62:c0:b8:63:f0:2e:30: + 0c:54:be:c9:94:7c:54:d4:40:06:6e:92:9d:72:2c:44:f5:e3: + c1:4a:3f:26:e4:21:ca:ba:ef:4f:a3:50:56:ee:89:5c:dc:5b: + 98:bc:0c:e9:db:16:98:f0:4c:3f:65:b9:d9:e0:2e:bd:83:9e: + c7:1b:f7:4f:87:5f:75:ff:67:5d:0a:ff:ce:fa:0b:f1:d5:8b: + 58:23:2b:92:27:b8:15:ff:66:c2:f9:68:e2:ed:64:a0:4a:e0: + 42:8b:ac:35:6a:67:64:f5:cb:a9:00:37:20:56:dd:3a:02:6e: + d8:0f:b4:5d:ff:92:a3:0d:65:fc:2e:ab:4e:37:bc:7e:d6:5b: + 1c:55:78:a0:6e:a9:2e:55:31:cf:61:f0:1b:4c:fe:fe:60:75: + 76:e8:d9:a1:f5:14:f0:43:33:4d:52:05:be:33:ff:da:31:94: + bd:0d:e5:4d:a8:ee:d1:f3:92:34:c2:36:91:37:3a:6d:ec:20: + df:b3:9e:a2:5d:d0:f0:a1:e2:ff:b7:85:ce:09:e3:c8:4a:6a: + 93:a4:ae:ec:c1:67:42:f7:8e:2f:64:ca:d7:76:72:bf:3f:ef: + 64:ff:10:dd:a2:3e:2d:f5:aa:02:cf:31:f0:6d:93:6b:56:4c: + b0:88:95:54:dc:d4:ac:dc:c1:88:27:8b:d9:9c:4e:66:93:f8: + dd:5e:74:41:26:9a:ef:e8:c8:46:5c:d5:f4:fc:cd:cc:67:c7: + 33:62:2e:70:9b:d1:39:de:34:fd:ad:9d:b9:48:61:e7:89:53: + 62:62:f0:40:6c:68:2f:e1:da:40:61:2a:50:af:c9:41:6d:98: + 16:af:cb:63:03:ec:04:99:e0:39:98:60:ae:aa:6a:57:9b:25: + 73:03:16:bf:78:c8:d7:2f:eb:fc:64:7f:c7:78:8c:34:15:5b: + 66:40:98:86:e6:0a -----BEGIN CERTIFICATE----- -MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt -3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY -IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh -P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd -pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW -9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo -TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o -8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa -IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB -wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ -jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst -gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw -AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD -VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN -JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 -+DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN -hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu -okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA -zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV -OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 -SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ -anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem.reference b/Lib/test/certdata/keycert3.pem.reference new file mode 100644 index 00000000000000..84d2ca29953e46 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem.reference @@ -0,0 +1,15 @@ +{'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2525 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem index ff4dceac7907c5..fcaa3a1656d1a5 100644 --- a/Lib/test/certdata/keycert4.pem +++ b/Lib/test/certdata/keycert4.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDGKA1zZDjeNPh2 -J9WHVXXMUf8h5N4/bHCM3CbIaZ1dShkCgfmFWmOtruEihgbfRYaSWZAwCmVAQGjm -gvUfgOIgsFfM8yO+zDByPhza7XvWPZfEe7mNRFe5ZlYntbeM/vuWCM4VzwDq/mqF -TFxNRmwInqE7hx0WnfCoQWe9N41hJyl1K0OjADb+SjlpJ0/UJ63hsB+dowGjaaBv -J8HduQcRqNg8s6FcyJJ8Mjss1uRMFK2j9QrmgbA61XuIPCxzc3J57mW8FN2KsR8D -2HOhe9nsTGlxp+O5Cudf/RBWB443xcoyduwRXOFTdEAU45MS4tKGP2hzezuxMFQn -LKARXVW4/gFxZk7kU8TweZUS6LAYPfYJnlfteb6z37LAbtoDvzKUKBEDf/nmoa7C -uKxSPC5HIKhLbjU/6kuPglSVEfJPJWu2bZJDAkFL85Ot3gPs10EX2lMUy0Jt3tf+ -TaQjEvFZhpKN8KAdYj3eVgOfzIBbQyjotHJjFe9Jkq4q7RoI+ncCAwEAAQKCAYAH -tRsdRh1Z7JmHOasy+tPDsvhVuWLHMaYlScvAYhJh/W65YSKd56+zFKINlX3fYcp5 -Fz67Yy+uWahXVE2QgFou3KX0u+9ucRiLFXfYheWL3xSMXJgRee0LI/T7tRe7uAHu -CnoURqKCulIqzLOO1efx1eKasXmVuhEtmjhVpcmDGv8SChSKTIjzgOjqT7QGE9Xq -eSRhq7mulpq9zWq+/369yG+0SvPs60vTxNovDIaBn/RHSW5FjeDss5QnmYMh/ukN -dggoKllQlkTzHSxHmKrIJuryZC+bsqvEPUFXN0NMUYcZRvt1lwdjzq/A+w4gDDZG -7QqAzYMYQZMw9PJeHqu4mxfUX5hJWuAwG5I2eV3kBRheoFw7MxP0tw40fPlFU+Zh -pRXbKwhMAlIHi0D8NyMn3hkVPyToWVVY3vHRknBB/52RqRq3MjqEFaAZfp0nFkiF -ytv3Dd5aeBb1vraOIREyhxIxE/qY8CtZC+6JI8CpufLmFXB412WPwl0OrVpWYfEC -gcEA486zOI46xRDgDw0jqTpOFHzh+3VZ8UoPoiqCjKzJGnrh2EeLvTsXX/GZOj0m -5zl6RHEGFjm5vKCh2C72Vj/m+AFVy7V9iJRzTYzP8So/3paaqo7ZaROTa6uStxdD -VPnY1uIgVQz9w5coN4dmr+RLBpFvvWaHp1wuC08YIWxcC9HSTQpbi1EP5eo08fOk -8reNkDEHxihDGHr1xW0z0qJqK1IVyLP7wDkmapudMZjkjqJSGJwwefV4qyGMTV2b -suW1AoHBAN6t9n6LBH553MF5iUrNJYxXh/SCom4Zft9aD6W4bZV/xL4XPpKBB4HX -aWdeI0iYZU9U+CZ88tBoQCt+JMrJ9cz03ENOvA/MBMREwbZ2hKmQgnoDZsV0vNry -6UsxeQmeNpGQFUz9foVJQVRdQCceN2YEABdehV1HZoSBbuGZkzqGJXrWwaf/ZhpB -dPYGUGOsczoD2/QLuWy2M7f7v0Ews6Heww3zipWzvdxKE0IpyVs30ZwVi8CRQiWU -bEcleXP6+wKBwAi3xEwJxV39Q1XQHuk+/fXywYMp/oMpXmfKUKypgBivUy0/r61S -MZbOXBrKdE6s+GzeFmmLU/xP+WGYinzKfUBIbMwa6e7sH218UgjcoQ0Xnlugk9ld -kmqwajDvhvgdh5rRlIMsuBlgE33shJV+mxBpSGlrHw3cjTaJlFbTGsKpCO9B0jcG -pyEZUWVg+ZMASz6VYcLHj6nEKtufTjhlVsLJpWPE34F/rmSuB9n6C+UZeSLP91rz -dea2pfPf/TFfcQKBwF4DSj9Qx/vxzS7t9fXbuM+QoPitMpCTOQppRpPr0nA8uj6b -J7LIwPejj3+xsenTVWpx8DanqAgvC3CRWE05iQoYEupj0mhE9Xo7oSE81nOUbFHB -H+GbkKRLzA0P/Q7/egBouWWA3Kq/K9LHb+9UBYWPiM5U/K9OFs04rCyZHxylSCud -gbNA08Wf/xZjwgri4t8KhBF75bQtFJbHtY57Vkuv9d/tA4SCl1Tq/UiAxd86KMfi -HNeXPDsLd89t1eIOgwKBwQDJkqwZXkhwkhoNuHRdzPO/1f5FyKpQxFs+x+OBulzG -zuwVKIawsLlUR4TBtF7PChOSZSH50VZaBI5kVti79kEtfNjfAzg4kleHrY8jQ/eq -HludZ3nmiPqqlbH4MH8NWczPEjee6z4ODROsAe31pz3S8YQK7KVoEuSf0+usJ894 -FtzS5wl6POAXTo2QeSNg9zTbb6JjVYcq6KCTnflDm4YEvFKI+ARqAXQHxm05wEOe -DbKC6hxxQbDaNOvXEAda8wU= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC/uCI/4eVSCIrx +//GcCCxeQP16Mtf0wAuINBR9uTySF8sEVoS6rNBwLeudZXaHmcp3jM7N2e9GjI8d +dQccPApKBwjAxhdT4XeuUTep4VraKWjLTWDvABHfCVFB1tX3HusEWaS8n/pH+N4L +UfnxS3HnFqGC5ib2DSgFTCnLQSj82iUgACdf87iZTons+kRvqYsdwpsgBUnO+w+o +c5o7Nj1bJJlTREsY6kQGY94as4pCkmqhoBTC7/le9wFGmXipRYi3pfSTOqAQJfqW +1VlQ4Bbc3lN5L+ONTmFXOgTJ5qhaS8cglnYElJI1QQTagyWQMHuBDIMpr3Iv/5qU +UzdIMzoMrpX6ien3Gan/jyjjzvP95jJvGujQudw/myXK8VOUThVtstRDFA3cIoV4 +UqMODz8w6qYU0GPYFBq//1UdpDrk/fBPBe3MLJT2+sB/kTzwxxYsKqcdjPG2Bt3Y +2nk7hCzE0KuueqCQPk/3fomY1TDyTlWOriAS40EYue5Rhx9qr8kCAwEAAQKCAYAL +X7EoeQ5Rv4/+q2B6EUIQlWp0RW/qZTpJ6k0M6GBfimnQ6BEXwgjbnt3LiKlvggAw +93mNXNCFLvGOndK+KhGyMpiiVx4rK8Ud4lObEHODXdGJvh1yEF7/DF51uXkYIA1x +RKAxUIxYmLsTkNlzJzaqrv0F9wF4t28YYVxZYpQ76/Un4Np1JtBcx/wGwxIsTbKj +IVhynd2zGdHj/He5643YSmOOPQ73e25tsp8KlnwtVuJclFKm/fWdXKoGtjQiVq2b +bViG7WTqb+a+R3IrO8CE2vAQ1ZqklP4Pg33AU1XFogRweqSkXvBGy1GHSSXCO4uX +gORj/bo/gAZ/eeufwqxBxJMS4Kbmxes/oYEgJidoaeA6DZjcKr+J3rFUt9j6rpxt +abeKBiK6wTHkKAdQwoccDdKFROvL4ZEu4zpjkmiqDrtIZEqtB8YCvoMpteIqrBhr +stLCDrftsysBTNGNn0THb4H5wx8nkZPCVyvaBD4N4Skp5rapP2bbrPhigcOoR8EC +gcEA4ToVg2b09tLWpmCg6XXssT1SvvFcSaJ3jjRKMfpmP5epWFiZ21qAOCbYhMNu +2G21AyNd1W7oVi6epXsB2YBOlSI0Fr6RuN0fDwg5jZLxEkEqDsL2QKsgoKHtGfBn +gX9jLQ5Lsy5/0CRXYCfPjdccbr2J5Gm2fU/jhhCi9ji/6b2RbgrRkIkr2I/J6KbS +sO+p1d93CFNagLMbdDspugDlIgj52OBMYO60nDlNU8O2pkAcLQTpWVqEjmoo4Q4K +oyOXAoHBANnqB93sLCndEiZNth7cOOEhNKCWRDkpGwDBbn2Dj3kBaXg8eCTG7tiF +FPyNVydYg+u0vcPIjj9URsOfpHc9MLSYtKAgCL3rSk+OQnROeg9hCjmB5Ucg7NTR +gSh/flfWp/q1EDeOVAu8B9DuBX1209br4xi78LnBP5/UzRjhuFEmvCvOqVCMCI7Y +I1k3l4NhOk87fKq5dbtxvNTU2eFTtt1+YjYwZFY+AUmx3WBz+aFwygks3S4SvN1J +LtLVYhOznwKBwQDC047WixIuDLX3WDD5orOroeNZHsn5PFv1HBBuaS9XpSatMH9u ++ztc12WGetQAze2+GDLMNNMv8cX0WZKBBfd0FBFA93pwkn6Sb0fxyoFUjCAIguen +iyB/M3M5c/blUz+EMxCSoA+aCkW2/NkS1lhXBwgoGLXuclPbnbqKCQ8h74TEzwD2 +6WGPRNqgsOYifj7IrjR2dDwehlCiW6c9qhaLOX5+94+6beK4HO1iHzN5Xo3A97Wv +QJjX5McV3yKeemMCgcA4N3reEo52IlULUqL4JSH7WkCkaP+iq2sO79fcQ3Ng6S9X +WGo6OqPlcbevS5s/SEOILDGEb5na1pgG4YlhRYTqIjb+1CTNMgUSrwWP0asFiqhD +m7IVfnX6lS23z+Q9LuBY+hr76hjeihyOFsmNy3jtCh+lAt8gXK1YQ2LB14FgVhjX +SFI/uFCA4VuFKaVJvGx5gkQwGvY3bCkl0t9+lMUpMPCPQD6yTP6yD1OoDWNJ9bn5 +UfyhZS4Z/EY7F9dcc8sCgcEAi8yW4gNCZSsGQRLh+fyDCSdak4ahATAeiI24231u +jK5+ncK9ZoZvzO/h+V5bw7ha59rD8aGmrLqzBVKEMi0PAT38psNH9ZtRt9y29YQ8 +WpyHtODMbphfbH4SFH3nj3PldRtBcOt0EDdp/4s64A9qdOWRx/vlJhdg0vtWTy5s +koyyOHhTcplpyPPq0BetYKl9UlXPVjus/RT1XV/lz2YrE8E9UAq+/r7ECoMExaZt +8j8J8O3lTWPxNTaQuiPiHA7E -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:c6:28:0d:73:64:38:de:34:f8:76:27:d5:87:55: - 75:cc:51:ff:21:e4:de:3f:6c:70:8c:dc:26:c8:69: - 9d:5d:4a:19:02:81:f9:85:5a:63:ad:ae:e1:22:86: - 06:df:45:86:92:59:90:30:0a:65:40:40:68:e6:82: - f5:1f:80:e2:20:b0:57:cc:f3:23:be:cc:30:72:3e: - 1c:da:ed:7b:d6:3d:97:c4:7b:b9:8d:44:57:b9:66: - 56:27:b5:b7:8c:fe:fb:96:08:ce:15:cf:00:ea:fe: - 6a:85:4c:5c:4d:46:6c:08:9e:a1:3b:87:1d:16:9d: - f0:a8:41:67:bd:37:8d:61:27:29:75:2b:43:a3:00: - 36:fe:4a:39:69:27:4f:d4:27:ad:e1:b0:1f:9d:a3: - 01:a3:69:a0:6f:27:c1:dd:b9:07:11:a8:d8:3c:b3: - a1:5c:c8:92:7c:32:3b:2c:d6:e4:4c:14:ad:a3:f5: - 0a:e6:81:b0:3a:d5:7b:88:3c:2c:73:73:72:79:ee: - 65:bc:14:dd:8a:b1:1f:03:d8:73:a1:7b:d9:ec:4c: - 69:71:a7:e3:b9:0a:e7:5f:fd:10:56:07:8e:37:c5: - ca:32:76:ec:11:5c:e1:53:74:40:14:e3:93:12:e2: - d2:86:3f:68:73:7b:3b:b1:30:54:27:2c:a0:11:5d: - 55:b8:fe:01:71:66:4e:e4:53:c4:f0:79:95:12:e8: - b0:18:3d:f6:09:9e:57:ed:79:be:b3:df:b2:c0:6e: - da:03:bf:32:94:28:11:03:7f:f9:e6:a1:ae:c2:b8: - ac:52:3c:2e:47:20:a8:4b:6e:35:3f:ea:4b:8f:82: - 54:95:11:f2:4f:25:6b:b6:6d:92:43:02:41:4b:f3: - 93:ad:de:03:ec:d7:41:17:da:53:14:cb:42:6d:de: - d7:fe:4d:a4:23:12:f1:59:86:92:8d:f0:a0:1d:62: - 3d:de:56:03:9f:cc:80:5b:43:28:e8:b4:72:63:15: - ef:49:92:ae:2a:ed:1a:08:fa:77 + 00:bf:b8:22:3f:e1:e5:52:08:8a:f1:ff:f1:9c:08: + 2c:5e:40:fd:7a:32:d7:f4:c0:0b:88:34:14:7d:b9: + 3c:92:17:cb:04:56:84:ba:ac:d0:70:2d:eb:9d:65: + 76:87:99:ca:77:8c:ce:cd:d9:ef:46:8c:8f:1d:75: + 07:1c:3c:0a:4a:07:08:c0:c6:17:53:e1:77:ae:51: + 37:a9:e1:5a:da:29:68:cb:4d:60:ef:00:11:df:09: + 51:41:d6:d5:f7:1e:eb:04:59:a4:bc:9f:fa:47:f8: + de:0b:51:f9:f1:4b:71:e7:16:a1:82:e6:26:f6:0d: + 28:05:4c:29:cb:41:28:fc:da:25:20:00:27:5f:f3: + b8:99:4e:89:ec:fa:44:6f:a9:8b:1d:c2:9b:20:05: + 49:ce:fb:0f:a8:73:9a:3b:36:3d:5b:24:99:53:44: + 4b:18:ea:44:06:63:de:1a:b3:8a:42:92:6a:a1:a0: + 14:c2:ef:f9:5e:f7:01:46:99:78:a9:45:88:b7:a5: + f4:93:3a:a0:10:25:fa:96:d5:59:50:e0:16:dc:de: + 53:79:2f:e3:8d:4e:61:57:3a:04:c9:e6:a8:5a:4b: + c7:20:96:76:04:94:92:35:41:04:da:83:25:90:30: + 7b:81:0c:83:29:af:72:2f:ff:9a:94:53:37:48:33: + 3a:0c:ae:95:fa:89:e9:f7:19:a9:ff:8f:28:e3:ce: + f3:fd:e6:32:6f:1a:e8:d0:b9:dc:3f:9b:25:ca:f1: + 53:94:4e:15:6d:b2:d4:43:14:0d:dc:22:85:78:52: + a3:0e:0f:3f:30:ea:a6:14:d0:63:d8:14:1a:bf:ff: + 55:1d:a4:3a:e4:fd:f0:4f:05:ed:cc:2c:94:f6:fa: + c0:7f:91:3c:f0:c7:16:2c:2a:a7:1d:8c:f1:b6:06: + dd:d8:da:79:3b:84:2c:c4:d0:ab:ae:7a:a0:90:3e: + 4f:f7:7e:89:98:d5:30:f2:4e:55:8e:ae:20:12:e3: + 41:18:b9:ee:51:87:1f:6a:af:c9 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,9 +90,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 1C:70:14:B0:20:DD:08:76:A4:3B:56:59:FA:5F:34:F8:36:66:E8:56 + E4:3D:4C:DB:96:85:CC:90:ED:0F:58:1E:8D:4D:7D:34:6C:4E:11:10 X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -103,59 +103,59 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 75:14:e5:68:45:8d:ed:6c:f1:27:1e:0e:f3:35:ae:0e:60:c1: - 65:36:62:b8:07:78:e1:b9:8d:7a:50:70:af:06:c9:d4:ee:50: - ef:d2:76:b2:a2:b6:cb:dc:a6:18:b5:3d:d2:f7:eb:0e:ec:b7: - 95:cd:2e:b1:36:6f:a8:9f:b8:4d:ff:ce:8a:c4:8e:62:37:32: - 80:3e:05:4a:4d:39:87:69:09:00:e8:40:64:d2:9d:f9:1f:9f: - ab:67:1f:f9:c6:84:ba:7e:17:6c:8b:8d:08:ee:fb:8a:d7:cd: - 06:25:72:9f:4e:1a:c2:71:e1:1b:cf:a2:d7:1c:05:12:95:d6: - 49:4b:e9:95:95:89:cf:68:18:46:a3:ea:0d:9d:8e:ca:1c:28: - 55:49:6b:c0:4b:58:f5:42:b9:0a:ec:0e:6e:21:a4:ff:60:c0: - 1b:6e:40:72:d0:a5:c5:b5:db:4e:87:67:3a:31:70:cb:32:84: - 70:a9:e2:ff:e0:f2:db:cd:03:b4:85:45:d3:07:cc:0f:c7:49: - d8:c2:17:eb:73:f7:4a:c0:d9:8c:59:ef:c0:0a:ce:13:0b:84: - c9:aa:0d:11:14:b4:e5:74:aa:ec:18:de:5f:26:18:98:4a:76: - f0:7f:cd:e6:c4:b5:58:03:03:f5:10:01:5d:8f:63:88:ba:65: - d7:b4:7f:5a:1a:51:0e:ed:e5:68:fa:18:03:72:15:a1:ec:27: - 1f:ea:ac:24:46:18:6e:f1:97:db:4a:f4:d6:a1:91:a0:8c:b0: - 2f:be:87:3b:44:b0:8d:2a:89:85:5f:f2:d9:e3:2e:66:b2:88: - 98:04:2c:96:32:38:99:19:a9:83:fd:94:0c:dd:63:d4:1b:60: - 9d:43:98:35:ac:b4:23:38:de:7f:85:52:57:a0:37:df:a5:cf: - be:54:2c:3c:50:27:2b:d4:54:a9:9d:a3:d4:a5:b3:c0:ea:3d: - 0e:e2:70:6b:fb:cb:a5:56:05:ec:64:72:f0:1a:db:64:01:cb: - 5d:27:c4:a1:c4:63 + 63:e9:cd:0f:ec:dd:27:a0:fc:76:16:a1:f2:5f:d3:07:9a:3f: + 97:f4:11:e8:20:e9:3c:dd:16:4f:20:62:71:73:a9:46:3e:85: + 6e:44:be:9d:08:4b:eb:80:08:d5:1b:93:5f:4a:0f:df:b7:1b: + 38:e2:2b:5c:68:48:ea:0a:58:5c:36:6c:79:a4:1e:b5:7e:ef: + cb:8d:5d:bd:7d:4a:e6:4e:dd:0b:87:10:ff:01:0e:9b:8b:bd: + de:1c:9a:25:fb:a2:e1:52:7b:8a:aa:08:37:b2:87:f7:45:9d: + 0b:ab:11:6b:0c:7f:db:ed:de:cc:1e:86:f0:be:30:25:6b:4b: + ff:f5:f2:99:ed:b0:b1:68:c1:44:0b:79:7b:81:95:b5:36:37: + 12:c4:99:9e:85:e8:2f:2d:cc:cd:d8:c6:f2:20:ec:6a:06:fd: + b8:fc:ff:02:11:95:4d:38:0d:42:8b:b2:43:eb:c2:b9:a4:e0: + 33:f1:da:25:2f:11:cf:57:b9:25:e5:ab:92:f2:b2:b5:11:2f: + bd:31:f7:55:eb:96:94:78:5a:ad:9c:8f:56:ec:34:a0:73:9c: + 01:c2:76:24:e0:f8:b3:c9:23:b9:ea:ab:c3:0a:d3:f8:53:44: + c7:12:37:76:44:d7:ee:25:93:f4:1c:1d:7c:fe:06:2c:fa:c5: + bb:c6:90:a4:57:fd:09:2e:da:af:f3:ac:e6:6d:7f:03:a1:26: + 36:9a:51:7d:8d:28:a7:5d:5d:37:57:cc:6a:11:2a:98:1d:57: + 71:32:6e:98:7c:12:28:ef:5c:2f:26:29:d4:56:0a:b1:23:d9: + 9d:35:20:1a:ec:cc:51:53:6e:ef:1c:e7:bc:3c:21:e7:64:31: + b1:4c:f3:55:b5:c4:93:17:d8:72:5a:05:2e:4a:e5:33:07:0e: + 7d:cf:22:73:0b:67:9f:b4:82:60:cd:71:f8:76:0c:c4:dc:98: + ee:49:f9:03:7f:0d:d8:c6:76:79:3c:28:ed:77:77:d3:7e:d5: + aa:1e:6e:2e:df:a5 -----BEGIN CERTIFICATE----- -MIIF9zCCBF+gAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF+TCCBGGgAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMYoDXNk -ON40+HYn1YdVdcxR/yHk3j9scIzcJshpnV1KGQKB+YVaY62u4SKGBt9FhpJZkDAK -ZUBAaOaC9R+A4iCwV8zzI77MMHI+HNrte9Y9l8R7uY1EV7lmVie1t4z++5YIzhXP -AOr+aoVMXE1GbAieoTuHHRad8KhBZ703jWEnKXUrQ6MANv5KOWknT9QnreGwH52j -AaNpoG8nwd25BxGo2DyzoVzIknwyOyzW5EwUraP1CuaBsDrVe4g8LHNzcnnuZbwU -3YqxHwPYc6F72exMaXGn47kK51/9EFYHjjfFyjJ27BFc4VN0QBTjkxLi0oY/aHN7 -O7EwVCcsoBFdVbj+AXFmTuRTxPB5lRLosBg99gmeV+15vrPfssBu2gO/MpQoEQN/ -+eahrsK4rFI8LkcgqEtuNT/qS4+CVJUR8k8la7ZtkkMCQUvzk63eA+zXQRfaUxTL -Qm3e1/5NpCMS8VmGko3woB1iPd5WA5/MgFtDKOi0cmMV70mSrirtGgj6dwIDAQAB -o4IBwzCCAb8wFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA4GA1UdDwEB/wQEAwIF -oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd -BgNVHQ4EFgQUHHAUsCDdCHakO1ZZ+l80+DZm6FYwfQYDVR0jBHYwdIAU8+yUjvKO -MMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZl -coIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6 -Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1Bggr -BgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2Nz -cC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5l -dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHUU5WhF -je1s8SceDvM1rg5gwWU2YrgHeOG5jXpQcK8GydTuUO/SdrKitsvcphi1PdL36w7s -t5XNLrE2b6ifuE3/zorEjmI3MoA+BUpNOYdpCQDoQGTSnfkfn6tnH/nGhLp+F2yL -jQju+4rXzQYlcp9OGsJx4RvPotccBRKV1klL6ZWVic9oGEaj6g2djsocKFVJa8BL -WPVCuQrsDm4hpP9gwBtuQHLQpcW1206HZzoxcMsyhHCp4v/g8tvNA7SFRdMHzA/H -SdjCF+tz90rA2YxZ78AKzhMLhMmqDREUtOV0quwY3l8mGJhKdvB/zebEtVgDA/UQ -AV2PY4i6Zde0f1oaUQ7t5Wj6GANyFaHsJx/qrCRGGG7xl9tK9NahkaCMsC++hztE -sI0qiYVf8tnjLmayiJgELJYyOJkZqYP9lAzdY9QbYJ1DmDWstCM43n+FUlegN9+l -z75ULDxQJyvUVKmdo9Sls8DqPQ7icGv7y6VWBexkcvAa22QBy10nxKHEYw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwM +ZmFrZWhvc3RuYW1lMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAv7gi +P+HlUgiK8f/xnAgsXkD9ejLX9MALiDQUfbk8khfLBFaEuqzQcC3rnWV2h5nKd4zO +zdnvRoyPHXUHHDwKSgcIwMYXU+F3rlE3qeFa2iloy01g7wAR3wlRQdbV9x7rBFmk +vJ/6R/jeC1H58Utx5xahguYm9g0oBUwpy0Eo/NolIAAnX/O4mU6J7PpEb6mLHcKb +IAVJzvsPqHOaOzY9WySZU0RLGOpEBmPeGrOKQpJqoaAUwu/5XvcBRpl4qUWIt6X0 +kzqgECX6ltVZUOAW3N5TeS/jjU5hVzoEyeaoWkvHIJZ2BJSSNUEE2oMlkDB7gQyD +Ka9yL/+alFM3SDM6DK6V+onp9xmp/48o487z/eYybxro0LncP5slyvFTlE4VbbLU +QxQN3CKFeFKjDg8/MOqmFNBj2BQav/9VHaQ65P3wTwXtzCyU9vrAf5E88McWLCqn +HYzxtgbd2Np5O4QsxNCrrnqgkD5P936JmNUw8k5Vjq4gEuNBGLnuUYcfaq/JAgMB +AAGjggHDMIIBvzAXBgNVHREEEDAOggxmYWtlaG9zdG5hbWUwDgYDVR0PAQH/BAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTkPUzbloXMkO0PWB6NTX00bE4REDB9BgNVHSMEdjB0gBTACiso +Q95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2Vy +dmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUG +CCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9v +Y3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3Qu +bmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAY+nN +D+zdJ6D8dhah8l/TB5o/l/QR6CDpPN0WTyBicXOpRj6FbkS+nQhL64AI1RuTX0oP +37cbOOIrXGhI6gpYXDZseaQetX7vy41dvX1K5k7dC4cQ/wEOm4u93hyaJfui4VJ7 +iqoIN7KH90WdC6sRawx/2+3ezB6G8L4wJWtL//Xyme2wsWjBRAt5e4GVtTY3EsSZ +noXoLy3MzdjG8iDsagb9uPz/AhGVTTgNQouyQ+vCuaTgM/HaJS8Rz1e5JeWrkvKy +tREvvTH3VeuWlHharZyPVuw0oHOcAcJ2JOD4s8kjueqrwwrT+FNExxI3dkTX7iWT +9BwdfP4GLPrFu8aQpFf9CS7ar/Os5m1/A6EmNppRfY0op11dN1fMahEqmB1XcTJu +mHwSKO9cLyYp1FYKsSPZnTUgGuzMUVNu7xznvDwh52QxsUzzVbXEkxfYcloFLkrl +MwcOfc8icwtnn7SCYM1x+HYMxNyY7kn5A38N2MZ2eTwo7Xd3037Vqh5uLt+l -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem index bd109921898044..d503d49dc85f15 100644 --- a/Lib/test/certdata/keycertecc.pem +++ b/Lib/test/certdata/keycertecc.pem @@ -1,8 +1,8 @@ -----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDRUbCeT3hMph4Y/ahL -1sy9Qfy4DYotuAP06UetzG6syv+EoQ02kX3xvazqwiJDrEyhZANiAAQef97STEPn -4Nk6C153VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn -4yLQbd/6YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qo= +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBuhPMBtySJ8hFhn5hU +1yvdDujmB8ajjXNlkQ9GhGdghQIAfKmP9vWtSY+1+BRjpAahZANiAAQKLvQoetPb +S5tbgExB5vSDUtgc7hK1j6AN++En6AkW8KkK+O/31jkItqBoSQz8ts+VCU5cloTf +H6i5AS9zUkHr4DEyn+XxfiutzcNQPdQIBmygWAsUpa1XlZf8ExMFf/A= -----END PRIVATE KEY----- Certificate: Data: @@ -13,19 +13,19 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc Subject Public Key Info: Public Key Algorithm: id-ecPublicKey Public-Key: (384 bit) pub: - 04:1e:7f:de:d2:4c:43:e7:e0:d9:3a:0b:5e:77:54: - 4c:76:e0:c3:64:25:47:26:2d:ee:fb:d6:ee:a5:af: - 74:3c:12:6d:c9:fd:83:da:03:58:bd:62:ce:ca:64: - 0d:d6:be:82:81:3e:85:9a:6a:29:27:e3:22:d0:6d: - df:fa:60:4e:80:02:33:2e:69:38:65:47:5f:dc:a8: - 7e:50:9e:64:b7:0d:71:d4:8e:6c:67:2e:35:b2:fd: - d6:65:d0:22:46:d6:aa + 04:0a:2e:f4:28:7a:d3:db:4b:9b:5b:80:4c:41:e6: + f4:83:52:d8:1c:ee:12:b5:8f:a0:0d:fb:e1:27:e8: + 09:16:f0:a9:0a:f8:ef:f7:d6:39:08:b6:a0:68:49: + 0c:fc:b6:cf:95:09:4e:5c:96:84:df:1f:a8:b9:01: + 2f:73:52:41:eb:e0:31:32:9f:e5:f1:7e:2b:ad:cd: + c3:50:3d:d4:08:06:6c:a0:58:0b:14:a5:ad:57:95: + 97:fc:13:13:05:7f:f0 ASN1 OID: secp384r1 NIST CURVE: P-384 X509v3 extensions: @@ -38,9 +38,9 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 45:ED:32:14:6D:51:A2:3B:B0:80:55:E0:A6:9B:74:4C:A5:56:88:B1 + F3:39:E5:90:3F:58:6F:55:9D:91:D4:86:B8:1B:14:35:09:AC:06:97 X509v3 Authority Key Identifier: - keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B Authority Information Access: @@ -51,53 +51,53 @@ Certificate: URI:http://testca.pythontest.net/testca/revocation.crl Signature Algorithm: sha256WithRSAEncryption Signature Value: - 07:e4:91:0b:d3:ed:4b:52:7f:50:68:c7:8d:80:48:9f:b7:4a: - 13:66:bf:9d:4c:2d:18:19:68:a0:da:3b:12:85:05:16:fa:8d: - 9c:58:c6:81:b3:96:ba:11:62:65:d3:76:f1:1c:ab:95:e4:d8: - 2a:e0:1f:7b:c5:20:2e:7c:8f:de:87:7a:2b:52:54:ca:d1:41: - b0:5e:20:72:df:44:00:4a:69:1a:ef:10:63:52:13:ed:49:02: - ee:dc:9d:f3:c8:ba:c4:01:81:5a:a9:1c:15:12:b6:21:de:44: - a5:fd:7e:f9:22:d1:3e:ee:22:dd:31:55:32:4e:41:68:27:c5: - 95:1b:7e:6b:18:74:f9:22:d6:b7:b9:31:72:51:a0:5a:2c:ff: - 62:76:e9:a0:55:8d:78:33:52:4a:58:b2:f4:4b:0c:43:82:2f: - a9:84:68:05:dd:11:47:70:24:fe:5c:92:fd:17:21:63:bb:fa: - 93:fa:54:54:05:72:48:ed:81:48:ab:95:fc:6d:a8:62:96:f9: - 3b:e2:71:18:05:3e:76:bb:df:95:17:7b:81:4b:1f:7f:e1:67: - 76:c4:07:cb:65:a7:f2:cf:e6:b4:fb:75:7c:ee:df:a1:f5:34: - 20:2b:48:fd:2e:49:ff:f3:a6:3b:00:49:6c:88:79:ed:9c:16: - 2a:04:72:e2:93:e4:7e:3f:2a:dd:30:47:9a:99:84:2a:b9:c4: - 40:31:a6:68:f3:20:d1:75:f1:1e:c8:18:64:5b:f8:4c:ce:9a: - 3c:57:2c:e3:63:64:29:0a:c2:b6:8e:20:01:55:9f:fe:10:ba: - 12:42:38:0a:9b:53:01:a5:b4:08:76:ec:e8:a6:fc:69:2c:f7: - 7f:5e:0f:44:07:55:e1:7c:2e:58:e5:d6:fc:6f:c2:4d:83:65: - bd:f3:32:e3:14:48:22:8d:80:18:ea:44:f8:24:79:ff:ff:c6: - 04:c2:e9:90:34:40:d6:59:3f:59:1e:4a:9a:58:60:ce:ab:f9: - 76:0e:ef:f7:05:17 + b8:d9:bf:b8:88:0a:01:2b:aa:64:32:e8:97:e1:0f:ee:34:40: + ef:71:fc:e5:f4:a2:3b:26:00:e2:19:3b:3e:cb:8e:2b:51:4c: + 30:ff:ab:44:9d:28:8d:d2:9c:32:e3:6b:96:73:1c:9d:55:76: + b2:bf:af:b8:51:ed:fd:04:e7:0d:fa:8c:2a:68:01:cb:92:90: + 7d:d1:31:d8:6c:f4:b8:4b:ea:62:59:1e:31:4e:f7:17:ae:3f: + f0:b6:ff:f8:6e:64:e3:6e:e9:19:4f:d0:1e:84:1e:df:56:49: + ae:90:a9:e0:7b:70:e6:97:7f:08:f2:03:49:82:7d:58:8e:a5: + a0:88:59:f3:a1:ab:b7:dd:6b:9a:2b:79:64:9c:d3:07:7c:ec: + a3:8d:56:77:28:10:c9:42:a5:fa:81:8e:35:ad:f7:28:da:58: + cc:a0:61:fa:0d:75:fd:26:ab:07:88:c5:64:21:f1:98:1c:e3: + 13:51:38:a7:59:a5:59:88:b9:10:4d:c1:79:70:3f:36:9e:08: + 75:53:7f:77:a3:3d:5d:16:b7:3b:a2:f6:eb:41:9b:56:16:80: + 34:96:57:f0:3a:3d:91:43:17:51:2b:64:2c:d7:e2:25:45:85: + f3:5f:73:f8:7d:aa:a3:6d:61:e8:ba:c5:90:5e:16:50:bc:79: + b9:4e:df:66:db:ae:95:80:15:da:4e:bf:8f:4e:9d:e3:7d:b0: + ab:8b:72:93:3c:56:1b:92:d3:67:07:d5:5a:ad:98:16:69:ea: + 46:fe:e0:d0:f4:ae:95:d3:b5:80:7d:f2:64:4d:14:c0:97:d2: + f3:91:bb:e6:43:d0:7e:39:14:0b:95:ab:c1:56:fd:26:79:f8: + 8c:69:eb:bc:74:0e:ea:cd:94:62:7e:30:58:01:7f:84:ee:9d: + 58:9c:fc:8f:75:1e:d9:86:cb:f4:fd:4c:af:b2:f7:69:ea:58: + cb:7c:93:ce:3d:86:f0:46:01:df:86:02:a1:6c:0f:fb:13:84: + d6:75:23:f9:17:21 -----BEGIN CERTIFICATE----- -MIIEyzCCAzOgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEzTCCAzWgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQef97STEPn4Nk6C153 -VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn4yLQbd/6 -YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qqjggHEMIIBwDAYBgNV -HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQURe0y -FG1RojuwgFXgppt0TKVWiLEwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/jmoZwMGf -dmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMstgJlaaVJb -MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 -cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww -OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAAfkkQvT7UtSf1Box42ASJ+3 -ShNmv51MLRgZaKDaOxKFBRb6jZxYxoGzlroRYmXTdvEcq5Xk2CrgH3vFIC58j96H -eitSVMrRQbBeIHLfRABKaRrvEGNSE+1JAu7cnfPIusQBgVqpHBUStiHeRKX9fvki -0T7uIt0xVTJOQWgnxZUbfmsYdPki1re5MXJRoFos/2J26aBVjXgzUkpYsvRLDEOC -L6mEaAXdEUdwJP5ckv0XIWO7+pP6VFQFckjtgUirlfxtqGKW+TvicRgFPna735UX -e4FLH3/hZ3bEB8tlp/LP5rT7dXzu36H1NCArSP0uSf/zpjsASWyIee2cFioEcuKT -5H4/Kt0wR5qZhCq5xEAxpmjzINF18R7IGGRb+EzOmjxXLONjZCkKwraOIAFVn/4Q -uhJCOAqbUwGltAh27Oim/Gks939eD0QHVeF8Lljl1vxvwk2DZb3zMuMUSCKNgBjq -RPgkef//xgTC6ZA0QNZZP1keSppYYM6r+XYO7/cFFw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEWMBQGA1UEAwwN +bG9jYWxob3N0LWVjYzB2MBAGByqGSM49AgEGBSuBBAAiA2IABAou9Ch609tLm1uA +TEHm9INS2BzuErWPoA374SfoCRbwqQr47/fWOQi2oGhJDPy2z5UJTlyWhN8fqLkB +L3NSQevgMTKf5fF+K63Nw1A91AgGbKBYCxSlrVeVl/wTEwV/8KOCAcQwggHAMBgG +A1UdEQQRMA+CDWxvY2FsaG9zdC1lY2MwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTz +OeWQP1hvVZ2R1Ia4GxQ1CawGlzB9BgNVHSMEdjB0gBTACisoQ95fyX1H5UebNvJl +jGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp +UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w +eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo +dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E +PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAuNm/uIgKASuqZDLol+EP +7jRA73H85fSiOyYA4hk7PsuOK1FMMP+rRJ0ojdKcMuNrlnMcnVV2sr+vuFHt/QTn +DfqMKmgBy5KQfdEx2Gz0uEvqYlkeMU73F64/8Lb/+G5k427pGU/QHoQe31ZJrpCp +4Htw5pd/CPIDSYJ9WI6loIhZ86Grt91rmit5ZJzTB3zso41WdygQyUKl+oGONa33 +KNpYzKBh+g11/SarB4jFZCHxmBzjE1E4p1mlWYi5EE3BeXA/Np4IdVN/d6M9XRa3 +O6L260GbVhaANJZX8Do9kUMXUStkLNfiJUWF819z+H2qo21h6LrFkF4WULx5uU7f +ZtuulYAV2k6/j06d432wq4tykzxWG5LTZwfVWq2YFmnqRv7g0PSuldO1gH3yZE0U +wJfS85G75kPQfjkUC5WrwVb9Jnn4jGnrvHQO6s2UYn4wWAF/hO6dWJz8j3Ue2YbL +9P1Mr7L3aepYy3yTzj2G8EYB34YCoWwP+xOE1nUj+Rch -----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py index 6626b93976a585..18e614496385fd 100644 --- a/Lib/test/certdata/make_ssl_certs.py +++ b/Lib/test/certdata/make_ssl_certs.py @@ -1,6 +1,7 @@ """Make the custom certificate and private key files used by test_ssl and friends.""" +import argparse import os import pprint import shutil @@ -8,7 +9,8 @@ from subprocess import * startdate = "20180829142316Z" -enddate = "20371028142316Z" +enddate_default = "25251028142316Z" +days_default = "140000" req_template = """ [ default ] @@ -79,8 +81,8 @@ default_startdate = {startdate} enddate = {enddate} default_enddate = {enddate} - default_days = 7000 - default_crl_days = 7000 + default_days = {days} + default_crl_days = {days} certificate = pycacert.pem private_key = pycakey.pem serial = $dir/serial @@ -117,7 +119,7 @@ here = os.path.abspath(os.path.dirname(__file__)) -def make_cert_key(hostname, sign=False, extra_san='', +def make_cert_key(cmdlineargs, hostname, sign=False, extra_san='', ext='req_x509_extensions_full', key='rsa:3072'): print("creating cert for " + hostname) tempnames = [] @@ -130,13 +132,13 @@ def make_cert_key(hostname, sign=False, extra_san='', hostname=hostname, extra_san=extra_san, startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) with open(req_file, 'w') as f: f.write(req) - args = ['req', '-new', '-nodes', '-days', '7000', + args = ['req', '-new', '-nodes', '-days', cmdlineargs.days, '-newkey', key, '-keyout', key_file, - '-extensions', ext, '-config', req_file] if sign: with tempfile.NamedTemporaryFile(delete=False) as f: @@ -145,7 +147,7 @@ def make_cert_key(hostname, sign=False, extra_san='', args += ['-out', reqfile ] else: - args += ['-x509', '-out', cert_file ] + args += ['-extensions', ext, '-x509', '-out', cert_file ] check_call(['openssl'] + args) if sign: @@ -175,7 +177,7 @@ def make_cert_key(hostname, sign=False, extra_san='', def unmake_ca(): shutil.rmtree(TMP_CADIR) -def make_ca(): +def make_ca(cmdlineargs): os.mkdir(TMP_CADIR) with open(os.path.join('cadir','index.txt'),'a+') as f: pass # empty file @@ -192,7 +194,8 @@ def make_ca(): hostname='our-ca-server', extra_san='', startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) t.write(req) t.flush() @@ -219,14 +222,22 @@ def make_ca(): shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') -def print_cert(path): +def write_cert_reference(path): import _ssl - pprint.pprint(_ssl._test_decode_cert(path)) + refdata = pprint.pformat(_ssl._test_decode_cert(path)) + print(refdata) + with open(path + '.reference', 'w') as f: + print(refdata, file=f) if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Make the custom certificate and private key files used by test_ssl and friends.') + parser.add_argument('--days', default=days_default) + parser.add_argument('--enddate', default=enddate_default) + cmdlineargs = parser.parse_args() + os.chdir(here) - cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') + cert, key = make_cert_key(cmdlineargs, 'localhost', ext='req_x509_extensions_simple') with open('ssl_cert.pem', 'w') as f: f.write(cert) with open('ssl_key.pem', 'w') as f: @@ -243,24 +254,26 @@ def print_cert(path): f.write(cert) # For certificate matching tests - make_ca() - cert, key = make_cert_key('fakehostname', ext='req_x509_extensions_simple') + make_ca(cmdlineargs) + cert, key = make_cert_key(cmdlineargs, 'fakehostname', ext='req_x509_extensions_simple') with open('keycert2.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('localhost', sign=True) + cert, key = make_cert_key(cmdlineargs, 'localhost', sign=True) with open('keycert3.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('fakehostname', sign=True) + check_call(['openssl', 'x509', '-outform', 'pem', '-in', 'keycert3.pem', '-out', 'cert3.pem']) + + cert, key = make_cert_key(cmdlineargs, 'fakehostname', sign=True) with open('keycert4.pem', 'w') as f: f.write(key) f.write(cert) cert, key = make_cert_key( - 'localhost-ecc', sign=True, key='param:secp384r1.pem' + cmdlineargs, 'localhost-ecc', sign=True, key='param:secp384r1.pem' ) with open('keycertecc.pem', 'w') as f: f.write(key) @@ -280,7 +293,7 @@ def print_cert(path): 'RID.1 = 1.2.3.4.5', ] - cert, key = make_cert_key('allsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'allsans', sign=True, extra_san='\n'.join(extra_san)) with open('allsans.pem', 'w') as f: f.write(key) f.write(cert) @@ -297,17 +310,17 @@ def print_cert(path): ] # IDN SANS, signed - cert, key = make_cert_key('idnsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'idnsans', sign=True, extra_san='\n'.join(extra_san)) with open('idnsans.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('nosan', sign=True, ext='req_x509_extensions_nosan') + cert, key = make_cert_key(cmdlineargs, 'nosan', sign=True, ext='req_x509_extensions_nosan') with open('nosan.pem', 'w') as f: f.write(key) f.write(cert) unmake_ca() - print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") - print_cert('keycert.pem') - print_cert('keycert3.pem') + print("Writing out reference data for Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") + write_cert_reference('keycert.pem') + write_cert_reference('keycert3.pem') diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem index c6ff8ea31bfa2e..c397342fa86f6e 100644 --- a/Lib/test/certdata/nosan.pem +++ b/Lib/test/certdata/nosan.pem @@ -1,131 +1,137 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC99xEYPTwFN/ji -i0lm11ckEGhcxciSsIgTgior54CLgQy7JXllTYmAWFTTg2zNBvDMexGI0h+xtZ4q -1Renghgt33N3Y6CT3v/L7JkE1abQbFveKW/ydlxH0+jLlsENSWjySwC80+f9L3bX -TcD8T4Fu9Uty2Rg1a/Eyekng5RmfkmLNgxfnX5R5nWhh0Aia7h3Ax2zCALfxqZIm -fxwavEgHsW/yZi+T+eoJwe0i7a6LaUoLqsPV9ZhagziNDaappPHH42NW39WlRhx1 -UjtiRm2Jihnzxcfs+90zLXSp5pxo/cE9Ia4d8ieq3Rxd/XgjlF6FXXFJjwfL36Dw -ehy8m3PKKAuO+fyMgPPPMQb7oaRy/MBG0NayRreTwyKILS2zafIW/iKpgICbxrWJ -r/H1b3S6PBKYUE2uQs0/ZPnRjjh0VeNnue7JcRoNbe27I2d56KUBsVEPdokjU59v -NYi6Se+ViZXtUbM1u/I0kvDMprAiobwtJFYgcE86N1lFJjHSwDMCAwEAAQKCAYBb -lvnJBA0iPwBiyeFUElNTcg2/XST9hNu2/DU1AeM6X7gxqznCnAXFudD8Qgt9NvF2 -xYeIvjbFydk+sYs8Gj9qLqhPUdukMAqI2cRVTmWla/lHPhdZgbOwdf1x23es3k4Z -NAxg/pKFwhK8cCKyA+tWAjKkZwODDk42ljt0kUEvbLbye1hVGAJQOJKRRmo/uLrj -rcNELnCBtc5ffT2hrlHUU7qz1ozt/brXhYa+JnbXhKZMxcKyMD2KtmXXrFNEy99o -jXbrpDCos82bzQfPDo8IpCbVbEd2J00aFmrNjQWhZuXX5dXflrujW4J0nzeHrZ78 -rNAz2/YuZ543BTB3XbogeFuLC5RqBgAMmw2WJ96Oa/UG8nZNvEw54N5r6dhfXj6A -VlJFLVwlfBQdAdaM3P4uZ6WECrH3EerQa27qyUdRrcDaGPLt7wG9FmMivnW1KQsy -5ow/gM0CsxFj2xNoGw1S5jtclbgSy8HNJaBsNk4XMQ+ORABZdG1MTTE+GMSjD/EC -gcEA+6JYiZEo+QrvItIZYB6Go4suu/F8df1pEjJlxwp2GmObitRhtV6r9g9IySFv -5SL7ZxARr4aQxvM7fNp57p9ssmkBtY0ofMjJAxhvs4T37bAcGK/2xCegNSmbqh24 -FAWpRDMgE5PjtuWC5jTvSOYFeUxwI/cu0HxWdxJl2dPUSL1nI2jP+ok3pZobEQk9 -E2+MlHpKmU+s/lAkuQiP+AW9a4M+ZJNWxocJjmtwj4OjJXPm7GddA/5x622DxFe6 -4K2vAoHBAMFC0An25bwGoFrCV/96s45K4qZcZcJ660+aK3xXaq6/8KfiusJnWds2 -nc0B6jYjKs8A7yTAGXke6fmyVsoLosZiXsbpW2m16g8jL79Tc85O9oDNmDIGk1uT -tRLZc2BvmHmy/dNrdbT/EHC3FKNWQVqWc2sHhPeB6F3hIEXDSUO/GB0njMZNXrPJ -547RlhN0xCLb3vTzzGHwNmwfI81YjV/XI4vpJjq1YceN8Xyd1r5ZOFfU8woIACO3 -I4dvBQ1avQKBwQCLDs9wzohfAFzg2Exvos7y6AKemDgYmD8NcE5wbWaQ9MTLNsz8 -RuIu64lkpRbKAMf/z5CGeI3fdCFGwRGq/e06tu7b3rMmKmtzS3jHM08zyiPsvKlZ -AzD00BaXLy8/2VUOPFaYmxy3QSRShaRKm9sgik5agcocKuo5iTBB7V8eB5VMqyps -IJJg8MXOZ1WaPQXqM56wFKjcLXvtyT6OaNWh6Xh8ajQFKDDuxI8CsFNjaiaONBzi -DSX1XaL4ySab7T8CgcEAsI+7xP6+EDP1mDVpc8zD8kHUI6zSgwUNqiHtjKHIo3JU -CO2JNkZ5v158eGlBcshaOdheozKlkxR9KlSWGezbf2crs4pKq585AS9iVeeGK3vU -lQRAAaQkSEv/6AKl9/q8UKMIZnkMhploibGZt0f8WSiOtb+e6QjUI8CjXVj2vF// -RdN2N01EMflKBh7Qf2H0NuytGxkJJojxD4K7kMVQE7lXjmEpPgWsGUZC01jYcfrN -EOFKUWXRys9sNDVnZjX5AoHAFRyOC1BlmVEtcOsgzum4+JEDWvRnO1hP1tm68eTZ -ijB/XppDtVESpq3+1+gx2YOmzlUNEhKlcn6eHPWEJwdVxJ87Gdh03rIV/ZQUKe46 -3+j6l/5coN4WfCBloy4b+Tcj+ZTL4sKaLm33RoD2UEWS5mmItfZuoEFQB958h3JD -1Ka1tgsLnuYGjcrg+ALvbM5nQlefzPqPJh0C8UV3Ny/4Gd02YgHw7Yoe4m6OUqQv -hctFUL/gjIGg9PVqTWzVVKaI +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAQKCAYAE +A1FQwmMxHOhUbuAIRoUInwhMeLs7ZKJx8OsGPDSCTGRGGnAFD/rSP63cNy71T+AF +gggpiGQegz9KkeGfhQwM1GssLGk8WnCp2fnqbAwyhkZzpOs+WEJSCsKMFDc3S0GU +hChIYAJFtgRLJnAFM+b2V0uIfHCz26hGqKOdDb/Wumzd8zPhH+/SNI5hdM7xStez +2d/dlqVZ21c0ghiMLZ/M2qrz8EEvWccc4BEuYtqJYNpdgAauLIGF0SxByPt/oZMf +4fa0zOVfAL6F5cPFkycRzuz97Lx+ZdtLuUBx/3y8Sk24TR1QutwaFhbwpRSQSUrg +xT7ZmS24rQhaP8i7+ROLGy0CPlLMc/D3WvvtdLDWOUF30azURvAFcqKA87Bp05bR +HI/ewXaSB26J34MUmBHy5dBjld+HX6hQInPhCTZu7gNDOrPaEc8Ug50c1r7HkYRl +VBv/4BQo3YtNdoKhL08MVuNbcxrzge3kI0mbQu7qDSyrve02WnQ1zijJ916NbgEC +gcEA6oFNNtWzVw/7/MBxUZ1tE5VW5U2SjRvQZlcPTJ9Ub16byhOCdNvE17uyYLt5 +diqoVAvSGWiJ+OKsmw3XTLdhxzWeZ1a9ZiCozsmIfHMQmL9t3MxzTnMEZCfcvxz0 +DPvAN0REUtwRkMbt7vC+g01Eom9UpAH7hKXkc3pTgtkoyFeebC/YneGtdOTVATKU +WMfsY6Biye7P1D8V7btjLI8LTnv90ygoP0uSrX53hp7k4EFAeXyT9VN5wMXLw81a +5K69AoHBANXZsvSSIwe7fjspNUU9yELnEokDIdBn4ARX6x5PonL2KB0iVUzWgjHb +vs23GAzAgKKY+S8VLYptBLUBlsZ/pSLfZLWd1ywxeHmk/nkzuqlAw1jwuUlMq6Vr +BJmfhpOoXeSrQwvkL/uOejdrDSmLAFiIV3l9LtkmIJzfQSSy2QB8mfWwni0vxOJT +mIqZE01rfRTNLIqxO3+0d7pDqArVvWxYqF4C8kihUhNjkB5NaURUTzDQ3Y33aAvT +zAm697nGSQKBwQDEQ8ed9ykL2sLpfR7aUclytHBvpYbcNsUqgf66ADeopiP48m8i +4rRSYjMepok3jugmv2XuAgJHnV8cvm7NNEXPdl7G2l/V08u0lhN3JM5lKQIH4801 +gSnRsVMdWFwhaaosFySfvLOu2e9VJYQtXEPvNwI96bLaCAW1aFHwl1N8qWhb34eK +S9DinopvYCesTlbX4uoLW6XxW4M83rJYHrg1zaxYR6m3n8Z5EflzYBTqY3JUuyES +F/U0k9bAX2SNNHkCgcALk8mYbADxfjkLQuPbZ8jbtl7OhBjki3sZQRk9ftowlxr8 +2Mr9ae+Ke3cM9AidSB6urtFutxrMD7LdicR74pUyGh39pxnrDpKTI1eTgDVuzE7H +FeEyErCIOA77siM7AzZyFsN+dVATslbzgRwpT5kpMdhqf1h18RZ656tDLVuKJzS+ +lF073QYvqo7rkfX1jwgqhCERMR8jfsWsk9UZIREsOHCFBmvPesxSuGUo/s/gHyBa +aDRWZzp+yWyWakTXDeECgcBaPIxLK0ky89Mv8jTSyIJIMhQj8+z8wFLJFQG5vF/Z +1taLIDY0stU8HzKflTJ5v72r2jpIvu5YgDEgl+kKjyqLMWSu4nBZ2OoJet49S6d3 +X0YL2VzsIiSG+8cmVvBBf7iZCHDKiCM20dzzhZq8BCLiaPq+Zu2No2xEJyE0m8rc +I66z6fV1BWQXLe6vPC8EVvBWz3Ybje0czUdZnZ44qV6qyZzREXuqbC0qS5RanE4Z ++Ps9b3TezE7gDII6SbUwhZo= -----END PRIVATE KEY----- Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: cb:2d:80:99:5a:69:52:61 Signature Algorithm: sha256WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:bd:f7:11:18:3d:3c:05:37:f8:e2:8b:49:66:d7: - 57:24:10:68:5c:c5:c8:92:b0:88:13:82:2a:2b:e7: - 80:8b:81:0c:bb:25:79:65:4d:89:80:58:54:d3:83: - 6c:cd:06:f0:cc:7b:11:88:d2:1f:b1:b5:9e:2a:d5: - 17:a7:82:18:2d:df:73:77:63:a0:93:de:ff:cb:ec: - 99:04:d5:a6:d0:6c:5b:de:29:6f:f2:76:5c:47:d3: - e8:cb:96:c1:0d:49:68:f2:4b:00:bc:d3:e7:fd:2f: - 76:d7:4d:c0:fc:4f:81:6e:f5:4b:72:d9:18:35:6b: - f1:32:7a:49:e0:e5:19:9f:92:62:cd:83:17:e7:5f: - 94:79:9d:68:61:d0:08:9a:ee:1d:c0:c7:6c:c2:00: - b7:f1:a9:92:26:7f:1c:1a:bc:48:07:b1:6f:f2:66: - 2f:93:f9:ea:09:c1:ed:22:ed:ae:8b:69:4a:0b:aa: - c3:d5:f5:98:5a:83:38:8d:0d:a6:a9:a4:f1:c7:e3: - 63:56:df:d5:a5:46:1c:75:52:3b:62:46:6d:89:8a: - 19:f3:c5:c7:ec:fb:dd:33:2d:74:a9:e6:9c:68:fd: - c1:3d:21:ae:1d:f2:27:aa:dd:1c:5d:fd:78:23:94: - 5e:85:5d:71:49:8f:07:cb:df:a0:f0:7a:1c:bc:9b: - 73:ca:28:0b:8e:f9:fc:8c:80:f3:cf:31:06:fb:a1: - a4:72:fc:c0:46:d0:d6:b2:46:b7:93:c3:22:88:2d: - 2d:b3:69:f2:16:fe:22:a9:80:80:9b:c6:b5:89:af: - f1:f5:6f:74:ba:3c:12:98:50:4d:ae:42:cd:3f:64: - f9:d1:8e:38:74:55:e3:67:b9:ee:c9:71:1a:0d:6d: - ed:bb:23:67:79:e8:a5:01:b1:51:0f:76:89:23:53: - 9f:6f:35:88:ba:49:ef:95:89:95:ed:51:b3:35:bb: - f2:34:92:f0:cc:a6:b0:22:a1:bc:2d:24:56:20:70: - 4f:3a:37:59:45:26:31:d2:c0:33 + 00:c3:e5:00:c7:04:14:ea:1b:98:3f:39:6e:a4:55: + be:a8:4c:72:cc:e2:ae:b0:5f:8d:98:a1:f3:c6:06: + 35:85:94:87:8c:5f:bf:00:d4:fc:d1:be:02:8b:a3: + d4:57:6c:13:9b:fd:0f:3d:ad:bb:6c:2f:d7:5d:f8: + 57:14:68:d3:fb:18:d2:e7:5f:e3:29:78:e9:cc:38: + be:8c:ec:bc:e2:84:58:b4:3b:24:66:bc:2c:ad:58: + 85:0c:62:41:fc:96:a1:ee:c9:dc:1d:51:13:ac:6e: + a1:a4:da:be:d2:6d:ef:17:cc:36:df:f5:d3:67:09: + 86:67:a0:e2:4c:88:6b:e9:d3:c0:f9:14:51:89:2f: + e2:bb:37:dc:42:10:70:a9:1c:9e:66:17:2c:b0:ce: + e9:e8:47:7b:ac:12:05:7e:d1:bf:27:b7:6f:3b:45: + c2:e8:04:d6:8e:44:22:ed:e8:bd:45:4a:c4:9b:ea: + 0d:cd:4b:f0:14:34:d0:db:d5:52:e8:6b:de:87:14: + b2:f7:ac:10:aa:4a:dd:f2:48:21:ad:cf:79:67:70: + 41:b9:1d:70:51:34:45:a8:91:ae:12:2a:a1:c3:38: + ff:33:0d:96:71:b6:63:70:5a:9e:90:b4:76:85:8f: + 8a:d8:fd:79:cc:1a:27:a4:26:02:06:72:bc:de:07: + c8:c0:83:01:82:f6:ed:b7:62:8e:7f:21:7e:4e:86: + 04:88:6e:a8:f4:9a:d6:15:47:06:70:25:bb:33:5d: + 54:27:ec:13:5b:43:33:45:36:09:e9:24:fa:49:c4: + 0e:31:0d:1a:f8:d7:92:2c:01:01:e6:ee:e8:2d:dc: + 6d:1a:5f:63:db:6d:76:31:3a:91:7f:51:df:b7:25: + 0b:b9:bd:81:9b:70:c5:31:a8:e3:da:1e:2d:c2:f1: + 05:13:ad:cb:c8:44:19:b8:5b:62:f6:7d:dd:f1:00: + da:42:e9:16:9f:45:55:e3:7d:e1:85:7e:4d:29:c8: + 0a:e5:bc:8c:98:08:6b:f1:01:e5 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 27:04:5B:37:51:0D:92:FD:2C:60:61:FE:4F:9C:66:20:41:2C:08:45 + X509v3 Authority Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 Signature Algorithm: sha256WithRSAEncryption Signature Value: - 7e:dd:64:64:92:6c:b9:41:ce:f3:e3:f8:e6:9f:c8:5b:32:39: - 8c:03:5b:5e:7e:b3:23:ca:6c:d1:99:2f:53:af:9d:3c:84:cd: - c6:ce:0a:ee:94:de:ff:a7:06:81:7e:e2:38:a5:05:39:58:22: - dc:13:83:53:e7:f8:16:cb:93:dc:cf:4b:e6:1b:9f:9e:71:ef: - ee:ba:ea:b6:68:5c:32:22:7e:54:4f:46:a6:0b:11:8f:ef:05: - 6e:d3:0b:d0:a8:be:95:23:a2:e4:e7:a8:a2:a4:7d:98:52:86: - a4:15:fb:74:7a:9a:89:23:43:20:26:3a:56:9e:a3:6e:54:02: - 76:4e:25:9c:a1:8c:03:99:e5:eb:a6:61:b4:9c:2a:b1:ed:eb: - 94:f9:14:aa:a4:c3:f0:f7:7a:03:a3:b1:f8:c0:83:79:ab:8a: - 93:7f:0a:95:08:50:ff:55:19:ac:28:a2:c8:9f:a6:77:72:a3: - da:37:a9:ff:f3:57:70:c8:65:d9:55:14:84:b4:b3:78:86:82: - da:84:2c:48:19:51:ec:9d:20:b1:4d:18:fb:82:9f:7b:a7:80: - 22:69:25:83:4d:bf:ac:31:64:f5:39:11:f1:ed:53:fb:67:ab: - 91:86:c5:4d:87:e8:6b:fe:9a:84:fe:6a:92:6b:62:c1:ae:d2: - f0:cb:06:6e:f3:50:f4:8d:6d:fa:7d:6a:1c:64:c3:98:91:da: - c9:8c:a9:79:e5:48:4c:a2:de:42:28:e8:0e:9f:52:6a:a4:e0: - c7:ac:11:9c:ba:5d:d6:84:93:56:28:f1:6d:83:aa:62:b2:b7: - 56:c6:64:d9:96:4e:97:ab:4e:8f:ba:f6:ab:b9:17:52:98:32: - 7f:b5:12:fa:39:d7:34:2a:f3:ed:40:90:6f:66:7b:b6:c1:9d: - b9:53:d0:e3:e9:69:8c:cf:7a:fd:08:0a:62:47:d4:ce:72:f7: - 6f:80:b4:1d:18:7a:ba:a2:a9:45:49:ef:9c:0b:99:89:03:ab: - 5f:7a:9d:c5:77:b7 + a0:b7:48:56:c6:b3:92:26:3f:d8:92:3c:ed:72:b5:89:ea:fd: + c9:66:da:ba:6a:8e:0d:04:ea:b2:fd:1f:e4:29:da:1e:c7:8f: + 5a:f0:88:74:dd:b3:f0:5e:a7:c4:77:13:cf:a8:19:fb:f2:2d: + ee:47:b4:0c:7c:5b:d3:dc:2f:2a:5c:bf:43:22:1c:91:d8:03: + 7d:44:90:c0:2d:fe:9e:7c:8b:ef:39:4e:b3:87:99:c8:eb:c2: + b7:cf:86:65:05:52:8c:15:b9:6a:8d:cd:e3:2a:29:d1:f5:87: + 42:11:c3:2e:42:ec:ed:26:55:8d:f3:ad:66:f4:79:72:f7:9e: + ed:bc:0c:5a:a7:74:ab:dc:57:e8:5c:99:b6:32:8f:7e:58:6e: + 70:48:ea:5d:a7:fa:b1:fc:c0:e9:50:3a:a4:53:21:e7:8b:77: + 54:2d:bb:64:1e:fc:88:86:90:c1:03:90:17:bb:3c:cf:ce:e8: + 48:32:c2:07:e5:8e:4f:93:a9:1a:e2:f5:93:a8:01:9f:30:26: + 1e:ed:b5:62:e9:25:c5:b0:32:e1:fc:bd:d6:48:b4:70:a9:e2: + cd:f6:a9:42:cb:bb:24:39:b9:34:fc:b9:cb:09:01:f0:5e:7e: + ef:b5:59:d6:88:31:a9:4c:be:7d:5b:de:4d:ec:84:1b:a1:1b: + d8:7d:83:cb:f1:04:c9:f1:f3:a4:08:05:3c:b5:96:13:1d:37: + 8a:23:83:22:86:72:17:13:5e:e8:89:06:58:cd:89:42:71:12: + e5:47:fc:f7:6e:96:28:8f:19:b9:d7:86:5b:c5:62:14:e1:5b: + 06:e7:e0:66:7e:fc:b7:9e:a9:99:14:e5:0a:d6:df:8f:b5:a2: + 1a:74:54:30:f6:f4:bf:1b:43:1d:be:4f:38:92:55:10:7b:d8: + 4f:e0:33:0f:40:2e:58:ec:9c:78:1b:43:17:b3:cb:0b:f5:34: + e2:7e:11:a1:90:b6:3c:79:6a:0b:91:ce:0b:8d:d5:60:e4:6d: + c8:2a:3d:40:6d:17 -----BEGIN CERTIFICATE----- -MIIEJDCCAowCCQDLLYCZWmlSYTANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTgwODI5MTQyMzE2WhcNMzcxMDI4MTQyMzE2 -WjBbMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNV -BAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ4wDAYDVQQDDAVub3NhbjCC -AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAL33ERg9PAU3+OKLSWbXVyQQ -aFzFyJKwiBOCKivngIuBDLsleWVNiYBYVNODbM0G8Mx7EYjSH7G1nirVF6eCGC3f -c3djoJPe/8vsmQTVptBsW94pb/J2XEfT6MuWwQ1JaPJLALzT5/0vdtdNwPxPgW71 -S3LZGDVr8TJ6SeDlGZ+SYs2DF+dflHmdaGHQCJruHcDHbMIAt/GpkiZ/HBq8SAex -b/JmL5P56gnB7SLtrotpSguqw9X1mFqDOI0Npqmk8cfjY1bf1aVGHHVSO2JGbYmK -GfPFx+z73TMtdKnmnGj9wT0hrh3yJ6rdHF39eCOUXoVdcUmPB8vfoPB6HLybc8oo -C475/IyA888xBvuhpHL8wEbQ1rJGt5PDIogtLbNp8hb+IqmAgJvGtYmv8fVvdLo8 -EphQTa5CzT9k+dGOOHRV42e57slxGg1t7bsjZ3nopQGxUQ92iSNTn281iLpJ75WJ -le1RszW78jSS8MymsCKhvC0kViBwTzo3WUUmMdLAMwIDAQABMA0GCSqGSIb3DQEB -CwUAA4IBgQB+3WRkkmy5Qc7z4/jmn8hbMjmMA1tefrMjymzRmS9Tr508hM3Gzgru -lN7/pwaBfuI4pQU5WCLcE4NT5/gWy5Pcz0vmG5+ece/uuuq2aFwyIn5UT0amCxGP -7wVu0wvQqL6VI6Lk56iipH2YUoakFft0epqJI0MgJjpWnqNuVAJ2TiWcoYwDmeXr -pmG0nCqx7euU+RSqpMPw93oDo7H4wIN5q4qTfwqVCFD/VRmsKKLIn6Z3cqPaN6n/ -81dwyGXZVRSEtLN4hoLahCxIGVHsnSCxTRj7gp97p4AiaSWDTb+sMWT1ORHx7VP7 -Z6uRhsVNh+hr/pqE/mqSa2LBrtLwywZu81D0jW36fWocZMOYkdrJjKl55UhMot5C -KOgOn1JqpODHrBGcul3WhJNWKPFtg6pisrdWxmTZlk6Xq06PuvaruRdSmDJ/tRL6 -Odc0KvPtQJBvZnu2wZ25U9Dj6WmMz3r9CApiR9TOcvdvgLQdGHq6oqlFSe+cC5mJ -A6tfep3Fd7c= +MIIEbzCCAtegAwIBAgIJAMstgJlaaVJhMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowWzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEOMAwGA1UEAwwF +bm9zYW4wggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAaNCMEAw +HQYDVR0OBBYEFCcEWzdRDZL9LGBh/k+cZiBBLAhFMB8GA1UdIwQYMBaAFMAKKyhD +3l/JfUflR5s28mWMZzviMA0GCSqGSIb3DQEBCwUAA4IBgQCgt0hWxrOSJj/Ykjzt +crWJ6v3JZtq6ao4NBOqy/R/kKdoex49a8Ih03bPwXqfEdxPPqBn78i3uR7QMfFvT +3C8qXL9DIhyR2AN9RJDALf6efIvvOU6zh5nI68K3z4ZlBVKMFblqjc3jKinR9YdC +EcMuQuztJlWN861m9Hly957tvAxap3Sr3FfoXJm2Mo9+WG5wSOpdp/qx/MDpUDqk +UyHni3dULbtkHvyIhpDBA5AXuzzPzuhIMsIH5Y5Pk6ka4vWTqAGfMCYe7bVi6SXF +sDLh/L3WSLRwqeLN9qlCy7skObk0/LnLCQHwXn7vtVnWiDGpTL59W95N7IQboRvY +fYPL8QTJ8fOkCAU8tZYTHTeKI4MihnIXE17oiQZYzYlCcRLlR/z3bpYojxm514Zb +xWIU4VsG5+Bmfvy3nqmZFOUK1t+PtaIadFQw9vS/G0Mdvk84klUQe9hP4DMPQC5Y +7Jx4G0MXs8sL9TTifhGhkLY8eWoLkc4LjdVg5G3IKj1AbRc= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem index 0a48bf7d23539c..c2c8b1ecdcfc97 100644 --- a/Lib/test/certdata/pycacert.pem +++ b/Lib/test/certdata/pycacert.pem @@ -7,96 +7,96 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (3072 bit) Modulus: - 00:d0:a0:9b:b1:b9:3b:79:ee:31:2f:b8:51:1c:01: - 75:ed:09:59:4c:ac:c8:bf:6a:0a:f8:7a:08:f0:25: - e3:25:7b:6a:f8:c4:d8:aa:a0:60:07:25:b0:cf:73: - 71:05:52:68:bf:06:93:ae:10:61:96:bc:b3:69:81: - 1a:7d:19:fc:20:86:8f:9a:68:1b:ed:cd:e2:6c:61: - 67:c7:4e:55:ea:43:06:21:1b:e9:b9:be:84:5f:c2: - da:eb:89:88:e0:42:a6:45:49:2a:d0:b9:6b:8c:93: - c9:44:3b:ca:fc:3c:94:7f:dd:70:c5:ad:d8:4b:3b: - dc:1e:f8:55:4b:b5:27:86:f8:df:b0:83:cf:f7:16: - 37:bf:13:17:34:d4:c9:55:ed:6b:16:c9:7f:ad:20: - 4e:f0:1e:d9:1b:bf:8d:ba:cd:ca:96:ef:1e:69:cb: - 51:66:61:48:0d:e8:79:a3:59:61:d4:10:8d:e0:0d: - 3b:0b:85:b6:d5:85:12:dd:a5:07:c5:4b:fa:23:fa: - 54:39:f7:97:0b:c9:44:47:1e:56:77:3c:3c:13:01: - 0b:6d:77:d6:14:ba:44:f4:53:35:56:d9:3d:b9:3e: - 35:5f:db:33:9a:4f:5a:b9:38:44:c9:32:49:fe:66: - f6:1f:1a:97:aa:ca:4c:4a:f6:b8:5e:40:7f:fb:0b: - 1d:f8:5b:a1:dc:f8:c0:2e:d0:6d:49:f5:d2:46:d4: - 90:57:fe:92:81:34:ae:2d:38:bb:a8:17:0c:e1:e5: - 3f:e2:f7:26:05:54:50:f5:64:b3:1c:6e:44:ff:6f: - a9:b4:03:96:e9:0e:c2:88:d8:72:52:90:99:c6:41: - 0f:46:90:59:b8:3e:6f:d2:e2:9e:1d:36:82:95:d3: - 58:8a:12:f3:e2:d8:0d:20:51:23:f0:90:2d:9a:3e: - 7d:26:86:b2:a7:d7:f9:28:60:03:e3:77:c7:88:04: - c9:fe:89:77:70:10:4f:c3:a0:8a:3b:f4:ab:42:7b: - e3:14:92:d8:ae:16:d6:a1:de:7d + 00:e6:ba:e2:e4:c1:c2:0c:1c:3e:62:d8:b9:5c:57: + 2e:52:b8:83:c5:88:3a:e6:9a:7a:f5:64:16:33:eb: + 37:6e:2f:7b:f3:68:03:45:65:47:5d:71:10:59:ca: + 2b:1b:00:6c:81:14:61:f4:86:59:3f:ea:fd:78:37: + 16:9d:43:f1:c4:f6:69:8c:c5:29:06:88:9e:26:22: + 04:ac:04:d8:87:34:48:39:eb:6b:f2:0b:92:aa:c3: + 6e:63:51:51:6b:c2:ad:ff:5c:c8:2f:b2:1b:9c:20: + 8a:40:3e:a2:2f:6a:ea:c8:d9:37:43:5c:dc:ed:92: + e2:d9:40:d2:61:9f:71:8a:f5:ed:39:ba:a8:5e:3e: + b5:21:63:10:d8:6f:b4:e2:11:01:0b:10:e8:bb:fb: + 62:ef:48:55:bc:f5:d2:9c:ab:68:ae:95:25:19:f2: + 97:7d:1a:dc:66:ea:88:5e:86:e4:cb:cb:69:4d:5e: + b0:a3:fb:6c:31:e4:28:60:5e:90:f1:d4:2e:10:50: + e1:85:f0:0d:5c:bd:dd:45:24:08:19:3e:1c:93:66: + 8f:2b:da:53:7d:04:1c:0e:42:c4:68:5e:a6:cd:a9: + 18:ed:a7:cd:6a:d0:d1:86:ba:90:ff:b7:4c:de:c7: + 43:24:6d:c7:1c:6b:9c:81:e7:e1:1b:57:25:90:a9: + 0e:c9:56:f3:f6:6b:5e:2d:b4:2e:40:50:9b:42:63: + d2:d6:99:1c:38:dc:cf:2b:2c:a7:72:f1:c7:5e:63: + 34:76:48:f4:3e:88:13:9e:86:16:53:2f:74:fb:87: + 01:8d:22:a4:68:33:ee:13:6c:7a:06:14:54:56:17: + 57:57:98:34:d0:0b:66:09:e3:88:09:f8:a5:15:1c: + 10:73:d0:88:50:99:5e:18:65:3b:ff:31:27:1b:5e: + c6:aa:41:fd:2d:2f:18:a7:c0:f2:ab:c7:22:b5:0b: + 69:d2:73:d1:bb:d0:1c:3d:fa:a4:35:62:cd:33:86: + c7:a0:23:0f:b9:6a:d5:d2:6d:8d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 X509v3 Authority Key Identifier: - F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign Signature Algorithm: sha256WithRSAEncryption Signature Value: - 8b:00:54:72:b3:8d:eb:f3:af:34:9f:d6:60:ea:de:84:3f:8c: - 04:8f:19:a6:be:02:67:c4:63:c5:74:e3:47:37:59:83:94:06: - f1:45:19:e8:07:2f:d6:4e:4b:4f:a8:3d:c7:07:07:27:92:f4: - 7e:73:4f:8b:32:19:94:46:7a:25:c4:d9:c4:27:b0:11:63:3a: - 60:8b:85:e1:73:4f:34:3b:6b:a4:34:8c:49:8e:cd:cf:4f:b2: - 65:27:41:19:b0:fc:80:31:78:f2:73:6a:9b:7d:71:34:50:fc: - 78:a8:da:05:b4:9c:5b:3a:99:7a:6b:5d:ef:3b:d3:e9:3b:33: - 01:12:65:cf:5e:07:d8:19:af:d5:53:ea:f0:10:ac:c4:b6:26: - 3c:34:2e:74:ee:64:dd:1d:36:75:89:44:00:b0:0d:fd:2f:b3: - 01:cc:1a:8b:02:cd:6c:e8:80:82:ca:bf:82:d7:00:9d:d8:36: - 15:d2:07:37:fc:6c:73:1d:da:a8:1c:e8:20:8e:32:7a:fe:6d: - 27:16:e4:58:6c:eb:3e:f0:fe:24:52:29:71:b8:96:7b:53:4b: - 45:20:55:40:5e:86:1b:ec:c9:46:91:92:ee:ac:93:65:91:2e: - 94:b6:b6:ac:e8:a3:34:89:a4:1a:12:0d:4d:44:a5:52:ed:8b: - 91:ee:2f:a6:af:a4:95:25:f9:ce:c7:5b:a7:00:d3:93:ca:b4: - 3c:5d:4d:f7:b1:3c:cc:6d:b4:45:be:82:ed:18:90:c8:86:d1: - 75:51:50:04:4c:e8:4f:d2:d6:50:aa:75:e7:5e:ff:a1:7b:27: - 19:1c:6b:49:2c:6c:4d:6b:63:cc:3b:73:00:f3:99:26:d0:82: - 87:d3:a9:36:9c:b4:3d:b9:48:68:a8:92:f0:27:8e:0c:cd:15: - 76:42:84:80:c9:3f:a3:7e:b4:dd:d7:f8:ac:76:ba:60:97:3c: - 5a:1f:4e:de:71:ee:09:39:10:dd:31:d5:68:57:5d:1a:e5:42: - ba:0e:68:e6:45:d3 + bc:e3:56:22:03:e4:5c:b9:67:ff:94:bc:75:9c:00:85:b0:d5: + 9c:c4:c3:29:66:5e:8b:b2:a9:a6:30:86:71:1a:6b:f2:00:c5: + 82:ab:5f:50:04:2a:fb:ed:b8:4c:b9:00:1b:49:57:92:11:cd: + a2:bc:cb:0f:a8:b4:61:f8:14:ca:a0:ec:40:17:ba:55:a1:c4: + bc:a6:b2:5a:ef:f4:20:10:77:47:d0:a0:c5:58:b9:6c:b5:10: + 7b:85:4a:43:a3:fb:2c:01:b9:77:17:b0:be:a0:ee:ae:ae:4d: + 67:86:48:89:57:86:78:ea:3c:ed:f0:41:35:8d:71:68:55:f9: + f2:e9:ac:32:d4:c6:a2:ef:ec:54:e6:c4:8e:2c:fd:bd:aa:60: + 56:65:33:95:ea:10:c6:74:04:eb:2a:6e:9b:11:f6:61:00:aa: + fd:ec:f2:0b:b1:4b:11:cd:93:eb:df:98:ae:4c:b4:07:04:4a: + e5:ef:ff:52:58:75:f5:3e:a4:71:e1:4a:72:5c:a9:8f:d4:aa: + 88:f0:6a:71:b4:c3:00:5f:99:6e:d7:91:af:6c:98:0d:64:c2: + 24:c7:9e:05:11:68:5e:24:62:e3:2e:45:ec:a3:34:f2:a3:9d: + 4d:e5:32:18:2f:74:fc:11:f1:36:50:4f:a0:40:29:68:5c:43: + 4c:23:6c:5d:72:c4:ec:52:76:eb:dc:b2:bc:1f:a6:c4:06:66: + 9b:5c:c7:cc:ca:f2:d1:25:4f:de:a5:1f:8d:e4:0c:49:b6:cf: + 85:40:a1:b9:1f:c6:c7:19:15:07:63:34:93:d0:57:a0:5a:70: + ec:af:4a:1c:72:17:1d:74:a3:6c:31:45:0b:33:7a:a1:b8:46: + db:c7:0e:64:4c:6f:b7:99:04:82:43:1f:e0:59:d6:99:21:27: + 28:09:40:ae:fc:c4:23:aa:a0:0c:08:05:2a:92:1c:db:23:9e: + d1:d5:63:ae:39:13:a3:12:88:5a:43:3c:4a:6e:32:f0:84:9f: + f9:09:0c:91:e7:b8 -----BEGIN CERTIFICATE----- -MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 -CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh -Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 -VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o -eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT -NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 -0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB -D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J -d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO -MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD -ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO -S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn -QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ -rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd -2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 -tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y -kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 -SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO -aOZF0w== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem index a6bf7356f4f684..0248f985545e20 100644 --- a/Lib/test/certdata/pycakey.pem +++ b/Lib/test/certdata/pycakey.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDQoJuxuTt57jEv -uFEcAXXtCVlMrMi/agr4egjwJeMle2r4xNiqoGAHJbDPc3EFUmi/BpOuEGGWvLNp -gRp9Gfwgho+aaBvtzeJsYWfHTlXqQwYhG+m5voRfwtrriYjgQqZFSSrQuWuMk8lE -O8r8PJR/3XDFrdhLO9we+FVLtSeG+N+wg8/3Fje/Exc01MlV7WsWyX+tIE7wHtkb -v426zcqW7x5py1FmYUgN6HmjWWHUEI3gDTsLhbbVhRLdpQfFS/oj+lQ595cLyURH -HlZ3PDwTAQttd9YUukT0UzVW2T25PjVf2zOaT1q5OETJMkn+ZvYfGpeqykxK9rhe -QH/7Cx34W6Hc+MAu0G1J9dJG1JBX/pKBNK4tOLuoFwzh5T/i9yYFVFD1ZLMcbkT/ -b6m0A5bpDsKI2HJSkJnGQQ9GkFm4Pm/S4p4dNoKV01iKEvPi2A0gUSPwkC2aPn0m -hrKn1/koYAPjd8eIBMn+iXdwEE/DoIo79KtCe+MUktiuFtah3n0CAwEAAQKCAYAD -iUK0/k2ZRqXJHXKBKy8rWjYMHCj3lvMM/M3g+tYWS7i88w00cIJ1geM006FDSf8i -LxjatvFd2OCg9ay+w8LSbvrJJGGbeXAQjo1v7ePRPttAPWphQ8RCS+8NAKhJcNJu -UzapZ13WJKfL2HLw1+VbziORXjMlLKRnAVDkzHMZO70C5MEQ0EIX+C6zrmBOl2HH -du6LPy8crSaDQg8YxFCI7WWnvRKp+Gp8aIfYnR+7ifT1qr5o9sEUw8GAReyooJ3a -yJ9uBUbcelO8fNjEABf9xjx+jOmOVsQfig2KuBEi0qXlQSpilZfUdYJhtNke9ADu -Hui6MBn04D4RIzeKXV+OLjiLwqkJyNlPuxJ2EGpIHNMcx3gpjXIApAwc47BQwLKJ -VhMWMXS0EWhCLtEzf5UrbMNX+Io3J7noEUu6jxmJV1BKhrnlYeoo4JryN0DUpkSb -rOAOJLOkpfj7+gvqmWI4MT6SQXSr6BK+3m4J5bVSq4pej9uG5NR3Utghi5hF7DEC -gcEA3cYNPYPFSTj9YAR3GUZvwUPVL3ZEFcwjrIeg87JhuZOH/hSQ33SgeEoAtaqL -cLbimj7YzUYx3FOUCp/7yK/bAF1dhAbFab1yZE46Qv2Vi4e+/KEBBftqxyJl5KyV -vc/HE1dXZGZIO1X5Z5MX8nO3rz/YayiozYVmMibrbHxgTEDC4BrbWtPJQNkflWEb -FXNjkm0s2+J3kFANpL94NUKMGtArxQV3hWydGN8wS3Fn7LDnHDoM5mOt/naeKRES -fwwpAoHBAPDTKsKs2LEe4YFzO1EClycDelppjxh5pHSnzTWSq40aKx533SG4aLyI -DmghzoA3OmY0xpAy1WpT9FeiDNbYpiFCH3qBkArQR2QCu+WGUQ9tDoeN0C2Dje4e -Yix49BjcGSWzSNvh+tU9PzRc/9eVBMAQuaCm3yNEL+Z7hFTzkrCWK23+jP/OzIIC -XhnKdOveIYVAjlVgv8CoWIy3xhwXyqPAcstcPmlv9sDAYn37Ot7rGIS7e0WyQxvg -gxnOxFzKNQKBwQDOPOn/NNV5HKh0bHKdbKVs4zoT4zW515etUIvbVR4QSCSFonZ/ -d6PreVZjmvAFp+3fZ2aSrx6bOJZJszGhFfjhw/G9X9aiWO1SXnVL6yrxERIJOWkM -ORy5h0GegOjYFauaTvUUhxHRLEi9i0sPy5EcRpFqReuFBPNe3Fa/EoMzJl6TriYj -tyRHTCNU9XMMZbxJZYH8EgUCjY/Cj9SoIvTL0p+Bn23hBHqrsJLm9dWhhXnHBC0O -68/Y/lJi+l9rCtECgcEAt6PfTJovl0j8HxF23vyBtK9TQtSR2NERlh9LPZn9lViq -Hs66YndT7sg1bDSzWlRDBSMjc1xAH5erkJOzBLYqYNwiUvGvnH9coSfwjkMRVxkL -ZlS+taZGuZiTtmP5h2d3CaegXIQDGU5d/xkXwxYQjEF0u8vkBel+OVxg+cLPTjcF -IRhl/r98dXtGtJYM+LvnhcxHfVWMg2YcOBn/SPbfgGVFZEuQECjf2fYaZQUJzGkr -xjOM+gXIZN6cOjbQyA0tAoHADgR5/bMbcf6Jk0w56c/khFZz/pusne5cjXw5a6qq -fClAqnqjGBpkRxs7HoCR3aje0Pd0pCS93a6Wiqneo4x4HDrpo+pWR2KGAAF4MeO3 -3K94hncmiLAiZo8iqULLKCqJW2EGB2b7QzGpY7jCPiI1g80KuYPesf4ZohSfrr1w -DoqGoNrcIVdVmUgX47lLqIiWarbbDRY0Am9j58dovmNINYr5wCYGbeh2RuUmHr4u -E2bb0CdekSHf05HPiF9QpK1z +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQDmuuLkwcIMHD5i +2LlcVy5SuIPFiDrmmnr1ZBYz6zduL3vzaANFZUddcRBZyisbAGyBFGH0hlk/6v14 +NxadQ/HE9mmMxSkGiJ4mIgSsBNiHNEg562vyC5Kqw25jUVFrwq3/XMgvshucIIpA +PqIvaurI2TdDXNztkuLZQNJhn3GK9e05uqhePrUhYxDYb7TiEQELEOi7+2LvSFW8 +9dKcq2iulSUZ8pd9Gtxm6ohehuTLy2lNXrCj+2wx5ChgXpDx1C4QUOGF8A1cvd1F +JAgZPhyTZo8r2lN9BBwOQsRoXqbNqRjtp81q0NGGupD/t0zex0Mkbccca5yB5+Eb +VyWQqQ7JVvP2a14ttC5AUJtCY9LWmRw43M8rLKdy8cdeYzR2SPQ+iBOehhZTL3T7 +hwGNIqRoM+4TbHoGFFRWF1dXmDTQC2YJ44gJ+KUVHBBz0IhQmV4YZTv/MScbXsaq +Qf0tLxinwPKrxyK1C2nSc9G70Bw9+qQ1Ys0zhsegIw+5atXSbY0CAwEAAQKCAYBf +4VWcPjBHHA2Iwgr1Jn1nfqmzklL3tUZXZwoa9SoJrc3Sbmy9j8LCP9PNnEehZuGw +GipClPnNp/dA15OcMrnrYYKnLt9Hico+inBqk3Dvbnh9KSmoYcrHD4N13jr5juMD +dSjzOQ5kKNmKrPx0u/dpE2r1oUdlql5+bYN/ceSbHGtCTCDfWSun/iTn7DO8pdhL +IvGz/Fk2mlaWuYiV9lz//5Z1W+w73sesNNYKgf/d+F9/+VNqMXbanLdypJmTBNp9 +eoS3eLk3ycoiGHCnVNm28IjeElFkUOJKVXY39BoMbS/x1MUHjuZDxdOcEaCV9iOr +adg7d91srwJlHGGX3IkC+08J5OJZtfTEMcB3i9g2omOxLAFfapeRxZdD6zMdxWUG +PUNIiy7UKaHe7JJ+fftnk0QBu5DzyrdP0fryFNz9nySot+gzVlU9idTMnIyE73UJ +foPowKpmxDOYNw0ZGXBIP/lMRNXkQM765P1NJpCN0Ud+kBl7GdFMkI++GFjiH3kC +gcEA+VIsOliXn5xfU2zXZb2q/bPPlCBDw/EzIelRTFVqUC2kL3oh4NlDNWAFgcfX +rQUCCoY9rEKYdTQI0w47K4AkJYajnDG2zHj6fkaPcLweyEDa9Hw2ezDDiwz2uwH2 +4HW+JBZ0I7rRuSRXTSJrzFYYVU6tsr54Kl85l2bmL8V2XVYj4ko9uddsyH0F1Ejt +BYPBghr01uEHsO3nIcNMl10fS9eXdhMTr/haXdDxpFXjR5AO/ouQEyGrXk6LPJpn +rUE7AoHBAOzpN8lQDQlf0JcYFBvFd7JdrhpbaIdATmT0NM/Mh+4B8qy5qnIJBdeQ +9T46sHmHzvhtGHmnP+dckKJi1Y/8Lt6noWNYDRnmrDol046ce2Nj8gxDGBT1kG4n +GBoKkA1Gj5N5yWMB8cGTVzQ+tViRMkn9+jmxRaNsW1Y0j7lMtVz0hJOUscbUALV1 +fYGH7zPiqfHoKoz7UuB2OlCKOH0+V96lAO4EgKMQNcEKMs9+Tg13fHTAyodxy/CY +tjUcKpef1wKBwBE24EDjDw0BMf/Dmxe2QdEkkieLFsK3q60iu+9GUoHYtOZmS2KH +/cD4sUilsLmMh/iMDkQPkRE+l4FjESjOvzAsHK3TLOjvTXRckNja1FFFURjiXqyg +0E+QhJSi7RXQa2F4f2pcItDitnhn8QN5ylJRjWKzDf729jYC78/KlYKaSP393Ecx +nZw2LanboynnT/wYumD/xpUrx/Kn1mj5EAkfiKCpbomO30Zs/9I17+xoAPEIV9lK +UNfBGpIDozbuMwKBwFlZmAWf4FrRvSzPEv5qWjt2I2yjXufrs+VVSPm6LOXx7CGC +oKsDhiWH8UZ4AgjD1KZTFvECyBItEgt8dQkp1k95L1/1XHORURFZJNHbaJnSnv5K +67Ez8DXrHqbrpuqq2wmG3BIwMIqOVExK/kAZ+rp3REEv/5CkFEqN5kq/iIM3YSz7 +3pSbbm0Bk8UfjHKoIOowYqPrQZWQYWvwxV9O/PrmhlQ+dHmLaoqUmxcwjqV7k//A +mmG85GqoXcfoCJRI3wKBwBdnxBzg17TMFufuvX2Bc/M9MqL3+vlwH6SDdr+2yYKA +hiD8Ur2OwtDGHnV4m3NeA/Guyz32H4CzvFAnpzlvMow/dvfp9JUcpdeidhIBZy8V +D7VODIiCyyTAb3g1LK0+HTEHAVRFihbNXhub/P6NckFXw0MJJQOvNNsYQnJTUngY +oxqdt1HeAujEwBrRSfrOGE2K8FVJ/MYf4PmxTIocICBk1/BmNHsUeJ5yFUDBweh5 +UJN6yp5PiGwvW8WFl4waXw== -----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl index 431a831c4f196a..a10586d1f79345 100644 --- a/Lib/test/certdata/revocation.crl +++ b/Lib/test/certdata/revocation.crl @@ -1,14 +1,14 @@ -----BEGIN X509 CRL----- -MIICJjCBjwIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +MIICKDCBkQIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j -YS1zZXJ2ZXIXDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlqgDjAMMAoGA1Ud -FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQDMZ4XLQlzUrqBbszEq9I/nXK3jN8/p -VZ2aScU2le0ySJqIthe0yXEYuoFu+I4ZULyNkCA79baStIl8/Lt48DOHfBVv8SVx -ZqF7/fdUZBCLJV1kuhuSSknbtNmja5NI4/lcRRXrodRWDMcOmqlKbAC6RMQz/gMG -vpewGPX1oj5AQnqqd9spKtHbeqeDiyyWYr9ZZFO/433lP7GdsoriTPggYJJMWJvs -819buE0iGwWf+rTLB51VyGluhcz2pqimej6Ra2cdnYh5IztZlDFR99HywzWhVz/A -2fwUA91GR7zATerweXVKNd59mcgF4PZWiXmQMwcE0qQOMqMmAqYPLim1mretZsAs -t1X+nDM0Ak3sKumIjteQF7I6VpSsG4NCtq23G8KpNHnBZVOt0U065lQEvx0ZmB94 -1z7SzjfSZMVXYxBjSXljwuoc1keGpNT5xCmHyrOIxaHsmizzwNESW4dGVLu7/JfK -w40uGbwH09w4Cfbwuo7w6sRWDWPnlW2mkoc= +YS1zZXJ2ZXIXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWqAOMAwwCgYD +VR0UBAMCAQAwDQYJKoZIhvcNAQELBQADggGBAKN4S2g4wCeU1fO5TSckAwxgdzdh +pY28f4musnQt7l37MzB2gmJVDSCZfQyrUnfSEST15WEY7CZVyTlbsu6gYKK53Yej +j3ORBfGUgzaz62Hs8di7SrHDzWUlNCFa47YFWDmtj96KTX1AItnpkCCE58Wfpivp +Hu+YINFpi/2vI2nvP/xcfvgT3dXek9kyz+2jHmadxcn2VerSBZZ9fiZk/k4NzgoI +JdiSswtN1c5GelHQfftwRXbWqsp6TvgHC5MagDuHh5Bj7/DftI7nCy0IT5GnP8lS +ZqmXUMpa8zbtSNSTIk0XepmypNW8HHMQbfJp0y7yOQ4pPyXICrjYTg7wKpODRcm3 +BRN89vvNfCszMU41glVfQG+2Po5uAMTl1hX8WYSj0+Xxrdg+wgJead4S5Sq3CgMT +bKsH2Dqh43L8BTxuxzLQyduK0gKSl8vlN7a9Bzm3IXYlyk+kKSyo4jP8XK79pj1k +1JglMFM9jpoMF2VmNjiROtVEl2tbDGwlvpjWYQ== -----END X509 CRL----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem index 427948252b786e..6db52404942210 100644 --- a/Lib/test/certdata/ssl_cert.pem +++ b/Lib/test/certdata/ssl_cert.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo -b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC -WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG -9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF -vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq -tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 -oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O -HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz -fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd -njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi -wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta -As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs -aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL -BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo -xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH -TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 -KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F -/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ -qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm -MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH -5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo -EGD2IF7VDQ== +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem index 6ab7d57d003a35..e60a66c2798784 100644 --- a/Lib/test/certdata/ssl_key.passwd.pem +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -1,42 +1,42 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIsc9l0YPybNICAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDxb9ekR9MERvIff73hFLc6BIIH -ENhkFePApZj7ZqpjBltINRnaZhu8sEfG1/y3ejDBOa5Sq3C/UPykPfJh0IXsraAB -STZO22UQEDpJzDnf1aLCo2cJpdz4Mr+Uj8OUdPiX83OlhC36gMrkgSYUdhSFQEas -MLiBnXU6Z5Mv1Lxe7TJrnMyA4A8JYXXu5XVTErJrC0YT6iCPQh7eAoEtml9a/tJM -OPg6kn58zmzVDp8LAau4Th1yhdD/cUQM09wg2i5JHLeC9akD+CkNlujVoAirLMTh -xoMXTy2dkv/lIwI9QVx6WE/VKIngBAPIi3Q+YCIm0PaTgWj5U10C8j4t7kW2AEZK -z82+vDOpLRGLo/ItNCO9F/a9e4PK4xxwFCOfR80tQNhs5gjKnbDz5IQv2p+pUfUX -u+AIO0rBb3M9Yya1MC2pc5VLAeQ3UF6YPrNyNjoDsQOytY3YtRVyxiKW72QzeUcX -Vpc3U6u8ZyHhkxK6bMv3dkPHGW1MOBd9/U5z+9lhHOfCGFStIQ9M8N48ZCWEGyty -oZT3UApxgqiBAi1h14ZyagA2mjsMNtTmmkSa3v26WUfrwnjm7LD1/0Vm+ptBOFH2 -CkP/aAvr8Ie+ehWobXGpqwB6rlOAwdpPrePtEZiZtdt58anmCquRgE5GIYtVz30f -flRABM8waJ196RDGkNAmDA3p/sqHy4vbsIOMl8faZ3QxvGVZlPbUEwPhiTIetA5Q -95fT/uIcuBLfpbaN23j/Av3LiJAeABSmGZ+dA+NXC5UMvuX8COyBU0YF2V6ofpIu -gP3UC7Tn4yV3Pbes81LEDCskaN6qVRil47l0G+dNcEHVkrGKcSaRCN+joBSCbuin -Rol34ir9azh8DqHRKdVlLlzTmDQcOwmi0Vx0ASgBXx4UI3IfK45gLJVoz6dkUz+3 -GIPrnh5cw2DvIgIApwmuCQUXPbWZwUW0zuyzhtny9W6S72GUE/P5oUCV+kGYBsup -FNiAyR9+n/xUuzB5HqIosj4rX+M4il4Ovt+KaCO6/COi+YjAO/9EnSttu8OTxsXl -wvgblsT7Y1d+iUfmIVNGtbc5NX46ktrbGiqgPX7oR7YDy5/+FQlnPS1YL0ThUiAC -2RbItu6b0uUyfu2jfWaGqy+SiRZ81rLwKPU3vJSEPfooVcJTG49EE006ZC4TvRzu -fNkId+P+BxvhEpUM4+VKzfzViEPuzR1u/DuwLAavS7nr5qb+zaUq+Fte5vDQmjjC -fflT8hS0BGpYEGndeZT4k+mZunHgs3NVUQ4/HW0nflf1j6qAn4+yIB79dH9d/ubt -RyBG29K+rN0TI/kH9BQZfsAcbnmhpT/ud0mJfeHZ0Lknn6mdJ/k4LXN0T1IlLKz3 -cSleOWY3zjKaOsbuju1o5IiVIr+AF/w+M4nzzDX6DDVpBPAt9iUnDGqjh6mJ3QWQ -CyCJDLNP0X8rZ8va2KOPorIBhmfDwJKEtIoXkb2hqWURTE0chC444QqiMsMXsX6+ -mOmiWGkdBFnEpGITISFTGERCjEfqOgTMweCANpquiLymJXgDURL603N2WexSgwnu -Gy1Ws1cA+1cT65ZLqjSqayZ6WdQvsKBBAnGW5LbwBhoCkX0vahs5nZiw0KnskP60 -wNMnyxaS1SuDJ65n+vuLUl7WeysRyz10RWliYZFiUE7jIXfWeYGonAo4eyCEeV/f -HInxxpswsg/na8BGBPMsx2SfBIiIvSIT4VNxHrL3sIfDrnb2HH/ut/oSLBgSKzY5 -DdkPz309kMM5dqnHANAgRrtVhqzLQE3kNGZ9mO/X1FAyXx8eB7NSeB6ysD8CAHvm -lkyfsGTzVsnuWWpeHqplds0wx5+XouVtFRI5J3RGa39mbpM1hMyIbS0O24CBKW6K -7n2UunbABwepL1hSa4e01OPdz4Zx/oayOevTtlfVqh68cEEc6ePdzf7z69pjot7B -eqlNaqa1POOmkuygL+fiP1BAR3rGEoQKXqb+6JjzLM9CnhCQHHPR2UdqukkEYwsa -bh9CU8AlfAJ19KFDria4JZXtl8LLMLLqWIO8fmQx7VqkEkEkl8jecO8YMaZTzFEb -bW7QtIZ1qHWH0UIHH3Qlav72NJTKvGIbtp1JNrLdsHcYNcojLZkEeA83UPaiTB2R -udltVUd016cktRVzLOKrust8kzPq3iSjpoIXFyFqIYHvWxGHgc7qD5gVBlazqSsV -qudDv+0PCBjLWLjS6HkFI8BfyXd3ME2wvSmTzSSgSh4nVJNNrZ/RVTtQ5MLVcdh0 -sJ3qsq2Pokf61XXjsTiQorX+cgI9zF6zETXHvnLf9FL+G/VSlcLUsQ0wC584qwQt -OSASYTbM79xgmjRmolZOptcYXGktfi2C4iq6V6zpFJuNMVgzZ+SbaQw9bvzUo2jG -VMwrTuQQ+fsAyn66WZvtkSGAdp58+3PNq31ZjafJXBzN +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQMMUN+qWFiwTbIjjb +cLEtYQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEGGzvsc6e5Zn2M9S +sdoMbaIEggcQ0mQYuMcujHtFfRKLmfclrf/dHOYgnVYc2c9MSR3jfoOwnTdBR1Op +MUuO4ukqrxGkrJBy+MLb1oPEGrnvn39pdBMZx4CBmdwX0jDq36UL9EU5/m071mT+ +Xoe/z/RHtrtmbpA9XKu7n/5gLi9jg57qorss8r5MaPHghm2ZOqaQZIOOMCF+4gNU +Cmp/NHL9TUM/sfQpaw9YM4VuTRyHNO3OiDKaCCz2romFrgM1TYz/I5/n5MMR543J +dsyIFzef3jHYfaXTl97v9ibGVpIUCKfHgYlQf41H+Me890AP4HaS0e2y+73/UwjB +4YSzPVm+lCiBWhFDbMuRmPY3YDQOA+TV06oCO8/aMamPRsryl8g7iOQAIz4hQdoP +ZTXtSZ+F68YmfPjKAj3NNQosvLzfuSfxZ9HqyvZ/0w1eead7pdSs7uvCEvN2Zcvk +a9EWy1bM895zD3DwrxqGnSYbitXouIOf9zHsl2lRUK44XmvgTHoNaJFIVyzVNgQr +e1YJM7LbEvErspa6cZTz7DCR44g9cYBi393XEt5yl4NCtq2PFq762pNqpsQqt5/Y +pLX5w12npf+OZ1WqJla89FabqLaJ1blmDEgfko0XriofW5gWLByvhYINtt+u5x/5 +QdTya2gWBxI2K+sAeGIt913RBAus8hLEDnVGDX/FEthz1OOKLXp0G6HQmqUom5PU +/O36GszhWIz1Q/SpMfZkdHfEfbJFCx0mpaJZTlC01hg+OpnwWxUvO2dxrK46xAzd +/obsVQRLOuyTxjIrQd+vVZRIcOroTHVUbh7qV2NV1sdQbsKnc4mYK8jjHsb/v/O3 +lcOFJX2V55z7FdEOi7qKKn3l5j6SNmdMGlGde16692VCdYo4Wjs2ufkYryJoJQ7L +O3AeoXVrpc2JwKj6GFKN3Bw6A3XEXTShWdHS8DAuHdbLBhHESeb/qA76qNInw/od +PXtsRNBWaS87rh7GGWrO2ULHBk0k5hLa9puXgAZVh5NcrVuujWM2spGPHGJWhvDt +ffOJAe0Kra8b1CHOC9aRUuHnapuu/mAD+XZTAt/UUNq0hdTDO1VTxL4lHpZjmWJ3 +OvmTp1HpA8FpSmzSBw6H0beVT9LXDbkSax86E2YQJPcC9IjQARoaL+7ba1XdXeSQ +k796zrq4YD1SB09qRVVar8nBSHUYAzG2gbAWgAQgOjwtC5nT1Que3kI7weG2nFZ4 +B1MfBmpEqPwMs8O5hurdYmfpBRoudivKrK5si+LB7og1IwVD2IS8mm55/Kmnugf4 +YCEiGu5s9F+njqEZBV7UPAabaFph6Iw3Jfmcg0YUZV60hY6O3EieonfwVBBKWBQp +K0SUs24Ld80B3oYWuqYI+MQlSprskRKFBfx7hp2PsQU9jAVPc8NohmfM5IJAFBwo +rCqAOsRsrRZnVoaRP89X50VuBg6ROiu/cmI5sqou3u30Pndzmlbcg8ekWj+1qrur +prva6uPc0oiCgFTMksFnhKvlHErIm+ceGvNSib+rWNomaL5cjpgnqAG6lZMP7N+p +QvH+ABchP+fJyyl/XuaOLmcXDSkIcadfjJSHACwmGRas3unfOYEyBg29oPu6PNlW +jXFJb7OzWaQfFNcCnHzvYGTvmrbg7VqBVqmVRYX3r6WNDJeaSwDJpxYNNQlTugbm +FOlD9JTeHyZ17rrqiCitbQwBVBpWOhBEIkFD+3JL1ZdGjyc3rcvbERLr/UN649vS +FcMOLiEvFjFdirq6Pe9fx/VD1GrIzXaEhvyCePSeoV6eILm7SSFTVCm+JMtiBlBi +ZjDxlUhQGuMg3IwZRqjfUR90wo3QWe5XOgM9mJa5qY/Yaa9YpFEJcd4mYnXcPLIz +eY8lqGAhROwJhGRhQoTQXucsSEhAkUBpRiE7UCN8OjezvAeFn0+6XyHeST+QZenX +ixAKJ27lFn/njvL4sDocd7ZvXpb3P5ZxCRakMnjunQQyjtUlJmPrNjs+ZlPenLuh +UA3Oj5d96dDqzgZNLxbDHKr6B+CMApBrwUDcum09PYgJ5xZ/Hrct4iSa57Gi528L +l0dcVgPHd80oIn/vyhjVYkkXNMrRhTJIgLuh0KsddZ7/8Xvxma9W8hNQ1sYwE0Yp +RqLgMRdFpTSN8hKUTfRvjzbrW4nQ2XVqAdyM6PEkdrPCBZczdGisp9oMF6woI/pA +ZKNxdUr0DG3tVBJ7z6qfdp2j+yHvhqQt+ohmk3YldPKpXfYz20ZJ52URE8vbCj2K +NZ/MVEACbg6FDgNQ1bIKD61pKL72+SjyKW1wQfIkeqMfOBI8BGclp7BL6dGRrVHN +PLg2j9gsgZ3XsiJOFtJ6Q3UABkeUrbRvAFQXPM8+keWU4VP/99dCJTvnEQTM6O8U +gz2NA4j1ZEOmB5L+hmf2gW+xgW6eeMjylj5JdeAliNaUqIhOju30vo4= -----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem index ee927210511dfc..1ad9e5e2b832cf 100644 --- a/Lib/test/certdata/ssl_key.pem +++ b/Lib/test/certdata/ssl_key.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 -qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M -V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz -hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td -ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID -eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb -SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV -uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ -HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO -M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP -MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ -tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP -qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq -D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph -7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq -U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG -ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC -gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ -u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG -XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s -5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo -dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH -Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 -UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr -r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 -XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 -1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx -tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz -og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX -v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z -SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 -uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 -+yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE -5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV -AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o -VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F -G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R -kNdcwhtZZ3dpChUdf+w/LIc= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index aef24e11393f6a..c81408b344968d 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1106,6 +1106,85 @@ def test_delta_non_days_ignored(self): dt2 = dt - delta self.assertEqual(dt2, dt - days) + def test_strptime(self): + inputs = [ + # Basic valid cases + (date(1998, 2, 3), '1998-02-03', '%Y-%m-%d'), + (date(2004, 12, 2), '2004-12-02', '%Y-%m-%d'), + + # Edge cases: Leap year + (date(2020, 2, 29), '2020-02-29', '%Y-%m-%d'), # Valid leap year date + + # bpo-34482: Handle surrogate pairs + (date(2004, 12, 2), '2004-12\ud80002', '%Y-%m\ud800%d'), + (date(2004, 12, 2), '2004\ud80012-02', '%Y\ud800%m-%d'), + + # Month/day variations + (date(2004, 2, 1), '2004-02', '%Y-%m'), # No day provided + (date(2004, 2, 1), '02-2004', '%m-%Y'), # Month and year swapped + + # Different day-month-year formats + (date(2004, 12, 2), '02/12/2004', '%d/%m/%Y'), # Day/Month/Year + (date(2004, 12, 2), '12/02/2004', '%m/%d/%Y'), # Month/Day/Year + + # Different separators + (date(2023, 9, 24), '24.09.2023', '%d.%m.%Y'), # Dots as separators + (date(2023, 9, 24), '24-09-2023', '%d-%m-%Y'), # Dashes + (date(2023, 9, 24), '2023/09/24', '%Y/%m/%d'), # Slashes + + # Handling years with fewer digits + (date(127, 2, 3), '0127-02-03', '%Y-%m-%d'), + (date(99, 2, 3), '0099-02-03', '%Y-%m-%d'), + (date(5, 2, 3), '0005-02-03', '%Y-%m-%d'), + + # Variations on ISO 8601 format + (date(2023, 9, 25), '2023-W39-1', '%G-W%V-%u'), # ISO week date (Week 39, Monday) + (date(2023, 9, 25), '2023-268', '%Y-%j'), # Year and day of the year (Julian) + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = date.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), date) + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit dates are allowed. + strptime = date.strptime + with self.assertRaises(ValueError): + # %y does require two digits. + newdate = strptime('01/02/3', '%d/%m/%y') + + d1 = date(2003, 2, 1) + d2 = date(2003, 1, 2) + d3 = date(2003, 1, 25) + inputs = [ + ('%d', '1/02/03', '%d/%m/%y', d1), + ('%m', '01/2/03', '%d/%m/%y', d1), + ('%j', '2/03', '%j/%y', d2), + ('%w', '6/04/03', '%w/%U/%y', d1), + # %u requires a single digit. + ('%W', '6/4/2003', '%u/%W/%Y', d1), + ('%V', '6/4/2003', '%u/%V/%G', d3), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + date.strptime('02-29', '%m-%d') + with self._assertNotWarns(DeprecationWarning): + date.strptime('20-03-14', '%y-%m-%d') + date.strptime('02-29,2024', '%m-%d,%Y') + class SubclassDate(date): sub_var = 1 @@ -2732,7 +2811,8 @@ def test_utcnow(self): def test_strptime(self): string = '2004-12-01 13:02:47.197' format = '%Y-%m-%d %H:%M:%S.%f' - expected = _strptime._strptime_datetime(self.theclass, string, format) + expected = _strptime._strptime_datetime_datetime(self.theclass, string, + format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) self.assertIs(type(expected), self.theclass) @@ -2746,8 +2826,8 @@ def test_strptime(self): ] for string, format in inputs: with self.subTest(string=string, format=format): - expected = _strptime._strptime_datetime(self.theclass, string, - format) + expected = _strptime._strptime_datetime_datetime(self.theclass, + string, format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) @@ -3342,6 +3422,9 @@ def test_fromisoformat_datetime_examples(self): ('2025-01-02T03:04:05,678+00:00:10', self.theclass(2025, 1, 2, 3, 4, 5, 678000, tzinfo=timezone(timedelta(seconds=10)))), + ('2025-01-02T24:00:00', self.theclass(2025, 1, 3, 0, 0, 0)), + ('2025-01-31T24:00:00', self.theclass(2025, 2, 1, 0, 0, 0)), + ('2025-12-31T24:00:00', self.theclass(2026, 1, 1, 0, 0, 0)) ] for input_str, expected in examples: @@ -3378,6 +3461,12 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:30:45.123456-05:00a', # Extra text '2009-04-19T12:30:45.123-05:00a', # Extra text '2009-04-19T12:30:45-05:00a', # Extra text + '2009-04-19T24:00:00.000001', # Has non-zero microseconds on 24:00 + '2009-04-19T24:00:01.000000', # Has non-zero seconds on 24:00 + '2009-04-19T24:01:00.000000', # Has non-zero minutes on 24:00 + '2009-04-32T24:00:00.000000', # Day is invalid before wrapping due to 24:00 + '2009-13-01T24:00:00.000000', # Month is invalid before wrapping due to 24:00 + '9999-12-31T24:00:00.000000', # Year is invalid after wrapping due to 24:00 ] for bad_str in bad_strs: @@ -3740,6 +3829,78 @@ def test_compat_unpickle(self): derived = loads(data, encoding='latin1') self.assertEqual(derived, expected) + def test_strptime(self): + # bpo-34482: Check that surrogates are handled properly. + inputs = [ + (self.theclass(13, 2, 47, 197000), '13:02:47.197', '%H:%M:%S.%f'), + (self.theclass(13, 2, 47, 197000), '13:02\ud80047.197', '%H:%M\ud800%S.%f'), + (self.theclass(13, 2, 47, 197000), '13\ud80002:47.197', '%H\ud800%M:%S.%f'), + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), self.theclass) + + def test_strptime_tz(self): + strptime = self.theclass.strptime + self.assertEqual(strptime("+0002", "%z").utcoffset(), 2 * MINUTE) + self.assertEqual(strptime("-0002", "%z").utcoffset(), -2 * MINUTE) + self.assertEqual( + strptime("-00:02:01.000003", "%z").utcoffset(), + -timedelta(minutes=2, seconds=1, microseconds=3) + ) + # Only local timezone and UTC are supported + for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), + (-_time.timezone, _time.tzname[0])): + if tzseconds < 0: + sign = '-' + seconds = -tzseconds + else: + sign ='+' + seconds = tzseconds + hours, minutes = divmod(seconds//60, 60) + tstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) + with self.subTest(tstr=tstr): + t = strptime(tstr, "%z %Z") + self.assertEqual(t.utcoffset(), timedelta(seconds=tzseconds)) + self.assertEqual(t.tzname(), tzname) + self.assertIs(type(t), self.theclass) + + # Can produce inconsistent time + tstr, fmt = "+1234 UTC", "%z %Z" + t = strptime(tstr, fmt) + self.assertEqual(t.utcoffset(), 12 * HOUR + 34 * MINUTE) + self.assertEqual(t.tzname(), 'UTC') + # yet will roundtrip + self.assertEqual(t.strftime(fmt), tstr) + + # Produce naive time if no %z is provided + self.assertEqual(strptime("UTC", "%Z").tzinfo, None) + + def test_strptime_errors(self): + for tzstr in ("-2400", "-000", "z"): + with self.assertRaises(ValueError): + self.theclass.strptime(tzstr, "%z") + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit times are allowed. + t = self.theclass(4, 5, 6) + inputs = [ + ('%H', '4:05:06', '%H:%M:%S', t), + ('%M', '04:5:06', '%H:%M:%S', t), + ('%S', '04:05:6', '%H:%M:%S', t), + ('%I', '4am:05:06', '%I%p:%M:%S', t), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = self.theclass.strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + def test_bool(self): # time is always True. cls = self.theclass @@ -4312,7 +4473,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt, t_rt + assert t == t_rt def test_fromisoformat_timespecs(self): time_bases = [ diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 7dcaf085a7ca91..d6be4ad049d14a 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -300,29 +300,78 @@ def get_build_info(): config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' cflags = sysconfig.get_config_var('PY_CFLAGS') or '' - cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' build = [] # --disable-gil if sysconfig.get_config_var('Py_GIL_DISABLED'): - build.append("free_threading") + if not sys.flags.ignore_environment: + PYTHON_GIL = os.environ.get('PYTHON_GIL', None) + if PYTHON_GIL: + PYTHON_GIL = (PYTHON_GIL == '1') + else: + PYTHON_GIL = None + + free_threading = "free_threading" + if PYTHON_GIL is not None: + free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" + build.append(free_threading) if hasattr(sys, 'gettotalrefcount'): # --with-pydebug build.append('debug') - if '-DNDEBUG' in (cflags + cflags_nodist): + if '-DNDEBUG' in cflags: build.append('without_assert') else: build.append('release') if '--with-assertions' in config_args: build.append('with_assert') - elif '-DNDEBUG' not in (cflags + cflags_nodist): + elif '-DNDEBUG' not in cflags: build.append('with_assert') + # --enable-experimental-jit + tier2 = re.search('-D_Py_TIER2=([0-9]+)', cflags) + if tier2: + tier2 = int(tier2.group(1)) + + if not sys.flags.ignore_environment: + PYTHON_JIT = os.environ.get('PYTHON_JIT', None) + if PYTHON_JIT: + PYTHON_JIT = (PYTHON_JIT != '0') + else: + PYTHON_JIT = None + + if tier2 == 1: # =yes + if PYTHON_JIT == False: + jit = 'JIT=off' + else: + jit = 'JIT' + elif tier2 == 3: # =yes-off + if PYTHON_JIT: + jit = 'JIT' + else: + jit = 'JIT=off' + elif tier2 == 4: # =interpreter + if PYTHON_JIT == False: + jit = 'JIT-interpreter=off' + else: + jit = 'JIT-interpreter' + elif tier2 == 6: # =interpreter-off (Secret option!) + if PYTHON_JIT: + jit = 'JIT-interpreter' + else: + jit = 'JIT-interpreter=off' + elif '-D_Py_JIT' in cflags: + jit = 'JIT' + else: + jit = None + if jit: + build.append(jit) + # --enable-framework=name framework = sysconfig.get_config_var('PYTHONFRAMEWORK') if framework: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 1722cc8612ca6b..8ef8fae44f1d25 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -26,7 +26,7 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locale, no_tracing, + TestFailed, run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget @@ -2895,7 +2895,7 @@ def test_float(self): got = self.loads(pickle) self.assert_is_copy(value, got) - @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '') def test_float_format(self): # make sure that floats are formatted locale independent with proto 0 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 628529b8664c77..f05be2b6bdf496 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -253,22 +253,16 @@ class USEROBJECTFLAGS(ctypes.Structure): # process not running under the same user id as the current console # user. To avoid that, raise an exception if the window manager # connection is not available. - from ctypes import cdll, c_int, pointer, Structure - from ctypes.util import find_library - - app_services = cdll.LoadLibrary(find_library("ApplicationServices")) - - if app_services.CGMainDisplayID() == 0: - reason = "gui tests cannot run without OS X window manager" + import subprocess + try: + rc = subprocess.run(["launchctl", "managername"], + capture_output=True, check=True) + managername = rc.stdout.decode("utf-8").strip() + except subprocess.CalledProcessError: + reason = "unable to detect macOS launchd job manager" else: - class ProcessSerialNumber(Structure): - _fields_ = [("highLongOfPSN", c_int), - ("lowLongOfPSN", c_int)] - psn = ProcessSerialNumber() - psn_p = pointer(psn) - if ( (app_services.GetCurrentProcess(psn_p) < 0) or - (app_services.SetFrontProcess(psn_p) < 0) ): - reason = "cannot run without OS X gui process" + if managername != "Aqua": + reason = f"{managername=} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -930,8 +924,8 @@ def check_sizeof(test, o, size): test.assertEqual(result, size, msg) #======================================================================= -# Decorator for running a function in a different locale, correctly resetting -# it afterwards. +# Decorator/context manager for running a code in a different locale, +# correctly resetting it afterwards. @contextlib.contextmanager def run_with_locale(catstr, *locales): @@ -942,16 +936,21 @@ def run_with_locale(catstr, *locales): except AttributeError: # if the test author gives us an invalid category string raise - except: + except Exception: # cannot retrieve original locale, so do nothing locale = orig_locale = None + if '' not in locales: + raise unittest.SkipTest('no locales') else: for loc in locales: try: locale.setlocale(category, loc) break - except: + except locale.Error: pass + else: + if '' not in locales: + raise unittest.SkipTest(f'no locales {locales}') try: yield @@ -959,6 +958,46 @@ def run_with_locale(catstr, *locales): if locale and orig_locale: locale.setlocale(category, orig_locale) +#======================================================================= +# Decorator for running a function in multiple locales (if they are +# availasble) and resetting the original locale afterwards. + +def run_with_locales(catstr, *locales): + def deco(func): + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + dry_run = '' in locales + try: + import locale + category = getattr(locale, catstr) + orig_locale = locale.setlocale(category) + except AttributeError: + # if the test author gives us an invalid category string + raise + except Exception: + # cannot retrieve original locale, so do nothing + pass + else: + try: + for loc in locales: + with self.subTest(locale=loc): + try: + locale.setlocale(category, loc) + except locale.Error: + self.skipTest(f'no locale {loc!r}') + else: + dry_run = False + func(self, *args, **kwargs) + finally: + locale.setlocale(category, orig_locale) + if dry_run: + # no locales available, so just run the test + # with the current locale + with self.subTest(locale=None): + func(self, *args, **kwargs) + return wrapper + return deco + #======================================================================= # Decorator for running a function in a specific timezone, correctly # resetting it afterwards. @@ -2209,7 +2248,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") @@ -2578,9 +2625,9 @@ def exceeds_recursion_limit(): return get_c_recursion_limit() * 3 -#Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') +# Windows doesn't have os.uname() but it doesn't support s390x. +is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' +skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') Py_TRACE_REFS = hasattr(sys, 'getobjects') @@ -2899,10 +2946,11 @@ def in_systemd_nspawn_sync_suppressed() -> bool: # trigger EINVAL. Otherwise, ENOENT will be given instead. import errno try: - with os.open(__file__, os.O_RDONLY | os.O_SYNC): - pass + fd = os.open(__file__, os.O_RDONLY | os.O_SYNC) except OSError as err: if err.errno == errno.EINVAL: return True + else: + os.close(fd) return False diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index edcd2b9a35bbd9..2b91bdcf9cd859 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -58,8 +58,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f493a92e0ddce8..14cd50bd30502c 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -849,7 +849,6 @@ def test_execution_namespace_is_main(self): ns.pop('__loader__') self.assertEqual(ns, { '__name__': '__main__', - '__annotations__': {}, '__doc__': None, '__package__': None, '__spec__': None, diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 0947464bb8c04e..e403c2a822788d 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -1,4 +1,4 @@ -from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, localeconv, Error) +from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) try: from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) except ImportError: @@ -74,6 +74,17 @@ def accept(loc): 'ps_AF': ('\u066b', '\u066c'), } +known_alt_digits = { + 'C': (0, {}), + 'en_US': (0, {}), + 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}), + 'ja_JP': (100, {0: '\u3007', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}), + 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}), + 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}), + 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}), + 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}), +} + if sys.platform == 'win32': # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) del known_numerics['ps_AF'] @@ -115,16 +126,17 @@ def numeric_tester(self, calc_type, calc_value, data_type, used_locale): def test_lc_numeric_nl_langinfo(self): # Test nl_langinfo against known values tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue for li, lc in ((RADIXCHAR, "decimal_point"), (THOUSEP, "thousands_sep")): if self.numeric_tester('nl_langinfo', nl_langinfo(li), lc, loc): tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') @@ -135,10 +147,10 @@ def test_lc_numeric_nl_langinfo(self): def test_lc_numeric_localeconv(self): # Test localeconv against known values tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue formatting = localeconv() @@ -146,6 +158,7 @@ def test_lc_numeric_localeconv(self): "thousands_sep"): if self.numeric_tester('localeconv', formatting[lc], lc, loc): tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') @@ -153,10 +166,10 @@ def test_lc_numeric_localeconv(self): def test_lc_numeric_basic(self): # Test nl_langinfo against localeconv tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue for li, lc in ((RADIXCHAR, "decimal_point"), @@ -173,6 +186,35 @@ def test_lc_numeric_basic(self): nl_radixchar, li_radixchar, loc, set_locale)) tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + def test_alt_digits_nl_langinfo(self): + # Test nl_langinfo(ALT_DIGITS) + tested = False + for loc, (count, samples) in known_alt_digits.items(): + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + with self.subTest(locale=loc): + alt_digits = nl_langinfo(locale.ALT_DIGITS) + self.assertIsInstance(alt_digits, tuple) + if count and not alt_digits and support.is_apple: + self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on Apple platforms') + self.assertEqual(len(alt_digits), count) + for i in samples: + self.assertEqual(alt_digits[i], samples[i]) + tested = True if not tested: self.skipTest('no suitable locales') @@ -180,10 +222,10 @@ def test_float_parsing(self): # Bug #1391872: Test whether float parsing is okay on European # locales. tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue @@ -199,6 +241,7 @@ def test_float_parsing(self): self.assertRaises(ValueError, float, localeconv()['decimal_point'].join(['1', '23'])) tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index dd8ceb55a411fb..eedf2506a14912 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1,12 +1,20 @@ """Tests for the annotations module.""" import annotationlib +import builtins import collections import functools import itertools import pickle import unittest -from annotationlib import Format, ForwardRef, get_annotations, get_annotate_function +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + get_annotate_function, + annotations_to_string, + value_to_string, +) from typing import Unpack from test import support @@ -24,16 +32,21 @@ def wrapper(a, b): return wrapper +class MyClass: + def __repr__(self): + return "my repr" + + class TestFormat(unittest.TestCase): def test_enum(self): - self.assertEqual(annotationlib.Format.VALUE.value, 1) - self.assertEqual(annotationlib.Format.VALUE, 1) + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) - self.assertEqual(annotationlib.Format.FORWARDREF.value, 2) - self.assertEqual(annotationlib.Format.FORWARDREF, 2) + self.assertEqual(Format.FORWARDREF.value, 2) + self.assertEqual(Format.FORWARDREF, 2) - self.assertEqual(annotationlib.Format.SOURCE.value, 3) - self.assertEqual(annotationlib.Format.SOURCE, 3) + self.assertEqual(Format.STRING.value, 3) + self.assertEqual(Format.STRING, 3) class TestForwardRefFormat(unittest.TestCase): @@ -41,9 +54,7 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) fwdref = anno["arg"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) self.assertEqual(fwdref.__forward_arg__, "x") @@ -53,16 +64,14 @@ def inner(arg: x): x = 1 self.assertEqual(fwdref.evaluate(), x) - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) self.assertIs(anno["x"], int) fwdref = anno["y"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) @@ -79,14 +88,14 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations(inner, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(inner, format=Format.STRING) self.assertEqual(anno, {"arg": "x"}) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) def test_expressions(self): @@ -120,7 +129,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -171,7 +180,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -205,7 +214,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -228,13 +237,13 @@ def f(fstring: f"{a}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) def f(fstring_format: f"{a:02d}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) class TestForwardRefClass(unittest.TestCase): @@ -263,7 +272,7 @@ class Gen[T]: with self.assertRaises(NameError): ForwardRef("T").evaluate(owner=int) - T, = Gen.__type_params__ + (T,) = Gen.__type_params__ self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) @@ -280,7 +289,13 @@ class Gen[T]: def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) - self.assertIs(ForwardRef("Counter", module="collections").evaluate(), collections.Counter) + self.assertIs( + ForwardRef("Counter", module="collections").evaluate(), collections.Counter + ) + self.assertEqual( + ForwardRef("Counter[int]", module="collections").evaluate(), + collections.Counter[int], + ) with self.assertRaises(NameError): # If globals are passed explicitly, we don't look at the module dict @@ -305,6 +320,36 @@ def test_fwdref_value_is_cached(self): self.assertIs(fr.evaluate(globals={"hello": str}), str) self.assertIs(fr.evaluate(), str) + def test_fwdref_with_owner(self): + self.assertEqual( + ForwardRef("Counter[int]", owner=collections).evaluate(), + collections.Counter[int], + ) + + def test_name_lookup_without_eval(self): + # test the codepath where we look up simple names directly in the + # namespaces without going through eval() + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) + self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) + with support.swap_attr(builtins, "int", dict): + self.assertIs(ForwardRef("int").evaluate(), dict) + + with self.assertRaises(NameError): + ForwardRef("doesntexist").evaluate() + + def test_fwdref_invalid_syntax(self): + fr = ForwardRef("if") + with self.assertRaises(SyntaxError): + fr.evaluate() + fr = ForwardRef("1+") + with self.assertRaises(SyntaxError): + fr.evaluate() + class TestGetAnnotations(unittest.TestCase): def test_builtin_type(self): @@ -333,22 +378,20 @@ class C1(metaclass=NoDict): self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(C1, format=Format.FORWARDREF), {"a": int}, ) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(C1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) self.assertEqual( - annotationlib.get_annotations( - NoDict, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(NoDict, format=Format.FORWARDREF), {"b": str}, ) self.assertEqual( - annotationlib.get_annotations(NoDict, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(NoDict, format=Format.STRING), {"b": "str"}, ) @@ -360,20 +403,20 @@ def f2(a: undefined): pass self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.VALUE), + annotationlib.get_annotations(f1, format=Format.VALUE), {"a": int}, ) self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) fwd = annotationlib.ForwardRef("undefined") self.assertEqual( - annotationlib.get_annotations(f2, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(f2, format=Format.FORWARDREF), {"a": fwd}, ) self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(f1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) @@ -396,30 +439,26 @@ def foo(): pass with self.assertRaises(ValueError): - annotationlib.get_annotations( - foo, format=annotationlib.Format.FORWARDREF, eval_str=True - ) - annotationlib.get_annotations( - foo, format=annotationlib.Format.SOURCE, eval_str=True - ) + annotationlib.get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + annotationlib.get_annotations(foo, format=Format.STRING, eval_str=True) def test_stock_annotations(self): def foo(a: int, b: str): pass - for format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + for format in (Format.VALUE, Format.FORWARDREF): with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), {"a": int, "b": str}, ) self.assertEqual( - annotationlib.get_annotations(foo, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(foo, format=Format.STRING), {"a": "int", "b": "str"}, ) foo.__annotations__ = {"a": "foo", "b": "str"} - for format in annotationlib.Format: + for format in Format: with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), @@ -441,10 +480,10 @@ def test_stock_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -479,7 +518,7 @@ def test_stock_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -513,48 +552,36 @@ def test_stock_annotations_in_module(self): ) self.assertEqual( - annotationlib.get_annotations(isa, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(isa, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.MyClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.MyClass, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function2, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function2, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function3, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function3, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - annotationlib, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(annotationlib, format=Format.STRING), {}, ) self.assertEqual( - annotationlib.get_annotations( - isa.UnannotatedClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.UnannotatedClass, format=Format.STRING), {}, ) self.assertEqual( annotationlib.get_annotations( - isa.unannotated_function, format=annotationlib.Format.SOURCE + isa.unannotated_function, format=Format.STRING ), {}, ) @@ -570,13 +597,11 @@ def test_stock_annotations_on_wrapper(self): {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations( - wrapped, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(wrapped, format=Format.FORWARDREF), {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations(wrapped, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(wrapped, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( @@ -593,12 +618,12 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.SOURCE}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, - {"format": annotationlib.Format.SOURCE, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -631,7 +656,7 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -705,17 +730,96 @@ def f(x: int): self.assertEqual(annotationlib.get_annotations(f), {"x": int}) self.assertEqual( - annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(f, format=Format.FORWARDREF), {"x": int}, ) f.__annotations__["x"] = str # The modification is reflected in VALUE (the default) self.assertEqual(annotationlib.get_annotations(f), {"x": str}) - # ... but not in FORWARDREF, which uses __annotate__ + # ... and also in FORWARDREF, which tries __annotations__ if available self.assertEqual( - annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF), - {"x": int}, + annotationlib.get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in STRING which always uses __annotate__ + self.assertEqual( + annotationlib.get_annotations(f, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + annotationlib.get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.FORWARDREF), {"x": int} + ) + + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.STRING), {"x": "int"} + ) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual( + annotationlib.get_annotations(hra, format=Format.VALUE), {"x": float} + ) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.STRING), {"x": str} ) def test_pep695_generic_class_with_future_annotations(self): @@ -845,15 +949,13 @@ def evaluate(format, exc=NotImplementedError): return undefined with self.assertRaises(NameError): - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.VALUE) + annotationlib.call_evaluate_function(evaluate, Format.VALUE) self.assertEqual( - annotationlib.call_evaluate_function( - evaluate, annotationlib.Format.FORWARDREF - ), + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), annotationlib.ForwardRef("undefined"), ) self.assertEqual( - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.SOURCE), + annotationlib.call_evaluate_function(evaluate, Format.STRING), "undefined", ) @@ -963,6 +1065,29 @@ class C: self.assertEqual(get_annotate_function(C)(Format.VALUE), {"a": int}) +class TestToSource(unittest.TestCase): + def test_value_to_string(self): + self.assertEqual(value_to_string(int), "int") + self.assertEqual(value_to_string(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_string(len), "len") + self.assertEqual(value_to_string(value_to_string), "value_to_string") + self.assertEqual(value_to_string(times_three), "times_three") + self.assertEqual(value_to_string(...), "...") + self.assertEqual(value_to_string(None), "None") + self.assertEqual(value_to_string(1), "1") + self.assertEqual(value_to_string("1"), "'1'") + self.assertEqual(value_to_string(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_string(MyClass()), "my repr") + + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) + self.assertEqual( + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} + ) + + class TestAnnotationLib(unittest.TestCase): def test__all__(self): support.check__all__(self, annotationlib) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ef05a6fefcffcc..78692fd3474782 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6,6 +6,7 @@ import io import operator import os +import py_compile import shutil import stat import sys @@ -15,10 +16,17 @@ import argparse import warnings -from test.support import os_helper, captured_stderr +from enum import StrEnum +from test.support import captured_stderr +from test.support import import_helper +from test.support import os_helper +from test.support import script_helper from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -380,15 +388,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -621,9 +636,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -679,7 +694,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -875,7 +890,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] @@ -914,6 +931,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" @@ -952,6 +986,34 @@ class TestDisallowLongAbbreviationAllowsShortGroupingPrefix(ParserTestCase): ] +class TestStrEnumChoices(TestCase): + class Color(StrEnum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + def test_parse_enum_value(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color) + args = parser.parse_args(['--color', 'red']) + self.assertEqual(args.color, self.Color.RED) + + def test_help_message_contains_enum_choices(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color, help='Choose a color') + self.assertIn('[--color {red,green,blue}]', parser.format_usage()) + self.assertIn(' --color {red,green,blue}', parser.format_help()) + + def test_invalid_enum_value_raises_error(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--color', choices=self.Color) + self.assertRaisesRegex( + argparse.ArgumentError, + r"invalid choice: 'yellow' \(choose from red, green, blue\)", + parser.parse_args, + ['--color', 'yellow'], + ) + # ================ # Positional tests # ================ @@ -1001,8 +1063,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1035,8 +1097,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1049,9 +1111,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1570,18 +1632,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] @@ -2156,20 +2224,33 @@ class TestNegativeNumber(ParserTestCase): argument_signatures = [ Sig('--int', type=int), Sig('--float', type=float), + Sig('--complex', type=complex), ] failures = [ '--float -_.45', '--float -1__000.0', + '--float -1.0.0', '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', ] successes = [ - ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0)), - ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0)), - ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0)), - ('--float -1_000.0', NS(int=None, float=-1000.0)), - ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0)), - ('--float -.5', NS(int=None, float=-0.5)), - ('--float -.5_000', NS(int=None, float=-0.5)), + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), ] class TestInvalidAction(TestCase): @@ -2238,14 +2319,14 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser @@ -2312,6 +2393,40 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + + def test_abbreviation(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('--foodle') + parser.add_argument('--foonly') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('bar') + parser1.add_argument('--fo') + parser1.add_argument('--foonew') + + self.assertEqual(parser.parse_args(['--food', 'baz', 'bar']), + NS(foodle='baz', foonly=None, fo=None, foonew=None)) + self.assertEqual(parser.parse_args(['--foon', 'baz', 'bar']), + NS(foodle=None, foonly='baz', fo=None, foonew=None)) + self.assertArgumentParserError(parser.parse_args, ['--fo', 'baz', 'bar']) + self.assertEqual(parser.parse_args(['bar', '--fo', 'baz']), + NS(foodle=None, foonly=None, fo='baz', foonew=None)) + self.assertEqual(parser.parse_args(['bar', '--foo', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertEqual(parser.parse_args(['bar', '--foon', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertArgumentParserError(parser.parse_args, ['bar', '--food', 'baz']) + def test_parse_known_args_with_single_dash_option(self): parser = ErrorRaisingArgumentParser() parser.add_argument('-k', '--known', action='count', default=0) @@ -2399,7 +2514,7 @@ def test_wrong_argument_subparsers_no_destination_error(self): parser.parse_args(('baz',)) self.assertRegex( excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$" + r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from foo, bar\)\n$" ) def test_optional_subparsers(self): @@ -2537,6 +2652,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2714,8 +2852,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2805,11 +2941,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2825,7 +2960,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2835,15 +2970,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2855,11 +2989,40 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) + def test_mutex_groups_parents(self): + parent = ErrorRaisingArgumentParser(add_help=False) + g = parent.add_argument_group(title='g', description='gd') + g.add_argument('-w') + g.add_argument('-x') + m = g.add_mutually_exclusive_group() + m.add_argument('-y') + m.add_argument('-z') + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) + + self.assertRaises(ArgumentParserError, parser.parse_args, + ['-y', 'Y', '-z', 'Z']) + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] + + options: + -h, --help show this help message and exit + + g: + gd + + -w W + -x X + -y Y + -z Z + ''')) + # ============================== # Mutually exclusive group tests # ============================== @@ -4586,7 +4749,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -4829,7 +4992,7 @@ class TestHelpNone(HelpTestCase): version = '' -class TestHelpTupleMetavar(HelpTestCase): +class TestHelpTupleMetavarOptional(HelpTestCase): """Test specifying metavar as a tuple""" parser_signature = Sig(prog='PROG') @@ -4856,6 +5019,34 @@ class TestHelpTupleMetavar(HelpTestCase): version = '' +class TestHelpTupleMetavarPositional(HelpTestCase): + """Test specifying metavar on a Positional as a tuple""" + + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('w', help='w help', nargs='+', metavar=('W1', 'W2')), + Sig('x', help='x help', nargs='*', metavar=('X1', 'X2')), + Sig('y', help='y help', nargs=3, metavar=('Y1', 'Y2', 'Y3')), + Sig('z', help='z help', nargs='?', metavar=('Z1',)), + ] + argument_group_signatures = [] + usage = '''\ + usage: PROG [-h] W1 [W2 ...] [X1 [X2 ...]] Y1 Y2 Y3 [Z1] + ''' + help = usage + '''\ + + positional arguments: + W1 W2 w help + X1 X2 x help + Y1 Y2 Y3 y help + Z1 z help + + options: + -h, --help show this help message and exit + ''' + version = '' + + class TestHelpRawText(HelpTestCase): """Test the RawTextHelpFormatter""" @@ -5196,15 +5387,15 @@ def custom_formatter(prog): class TestInvalidArgumentConstructors(TestCase): """Test a bunch of invalid Argument constructors""" - def assertTypeError(self, *args, **kwargs): + def assertTypeError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(TypeError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(TypeError, errmsg, parser.add_argument, + *args, **kwargs) - def assertValueError(self, *args, **kwargs): + def assertValueError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(ValueError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(ValueError, errmsg, parser.add_argument, + *args, **kwargs) def test_invalid_keyword_arguments(self): self.assertTypeError('-x', bar=None) @@ -5214,8 +5405,9 @@ def test_invalid_keyword_arguments(self): def test_missing_destination(self): self.assertTypeError() - for action in ['append', 'store']: - self.assertTypeError(action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + self.assertTypeError(action=action) def test_invalid_option_strings(self): self.assertValueError('--') @@ -5232,55 +5424,73 @@ def test_invalid_action(self): self.assertValueError('-x', action='foo') self.assertValueError('foo', action='baz') self.assertValueError('--foo', action=('store', 'append')) - parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: - parser.add_argument("--foo", action="store-true") - self.assertIn('unknown action', str(cm.exception)) + self.assertValueError('--foo', action="store-true", + errmsg='unknown action') + + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') with self.assertRaises(ValueError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: - for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: - self.assertTypeError('-x', action=action, **attrs) + with self.subTest(action=action): + for attrs in [dict(type=int), dict(nargs='+'), + dict(choices=['a', 'b'])]: + with self.subTest(attrs=attrs): + self.assertTypeError('-x', action=action, **attrs) + self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") + self.assertTypeError('-x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments for action in ['store_true', 'store_false', 'count']: + with self.subTest(action=action): + # const is always disallowed + self.assertTypeError('-x', const='foo', action=action) - # const is always disallowed - self.assertTypeError('-x', const='foo', action=action) - - # nargs is always disallowed - self.assertTypeError('-x', nargs='*', action=action) + # nargs is always disallowed + self.assertTypeError('-x', nargs='*', action=action) def test_more_than_one_argument_actions(self): - for action in ['store', 'append']: - - # nargs=0 is disallowed - self.assertValueError('-x', nargs=0, action=action) - self.assertValueError('spam', nargs=0, action=action) - - # const is disallowed with non-optional arguments - for nargs in [1, '*', '+']: - self.assertValueError('-x', const='foo', - nargs=nargs, action=action) - self.assertValueError('spam', const='foo', - nargs=nargs, action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + # nargs=0 is disallowed + action_name = 'append' if action == 'extend' else action + self.assertValueError('-x', nargs=0, action=action, + errmsg=f'nargs for {action_name} actions must be != 0') + self.assertValueError('spam', nargs=0, action=action, + errmsg='nargs for positionals must be != 0') + + # const is disallowed with non-optional arguments + for nargs in [1, '*', '+']: + self.assertValueError('-x', const='foo', + nargs=nargs, action=action) + self.assertValueError('spam', const='foo', + nargs=nargs, action=action) def test_required_const_actions(self): for action in ['store_const', 'append_const']: - - # nargs is always disallowed - self.assertTypeError('-x', nargs='+', action=action) + with self.subTest(action=action): + # nargs is always disallowed + self.assertTypeError('-x', nargs='+', action=action) def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers') @@ -6452,6 +6662,27 @@ def test_required_args(self): 'the following arguments are required: bar, baz$', self.parser.parse_args, []) + def test_required_args_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, BaZ$', + self.parser.parse_args, []) + + def test_required_args_n(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_n_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3, metavar=('B', 'A', 'Z')) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, B, A, Z$', + self.parser.parse_args, []) + def test_required_args_optional(self): self.parser.add_argument('bar') self.parser.add_argument('baz', nargs='?') @@ -6466,6 +6697,20 @@ def test_required_args_zero_or_more(self): 'the following arguments are required: bar$', self.parser.parse_args, []) + def test_required_args_one_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_one_or_more_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+', metavar=('BaZ1', 'BaZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'the following arguments are required: bar, BaZ1\[, BaZ2]$', + self.parser.parse_args, []) + def test_required_args_remainder(self): self.parser.add_argument('bar') self.parser.add_argument('baz', nargs='...') @@ -6481,12 +6726,55 @@ def test_required_mutually_exclusive_args(self): 'one of the arguments --bar --baz is required', self.parser.parse_args, []) + def test_conflicting_mutually_exclusive_args_optional_with_metavar(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='?', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BaZ: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BaZ$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar1(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1',)) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BAZ1: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BAZ1$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar2(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1', 'BAZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument BAZ1\[, BAZ2]: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument --bar: not allowed with argument BAZ1\[, BAZ2]$', + self.parser.parse_args, ['a', '--bar', 'b']) + def test_ambiguous_option(self): self.parser.add_argument('--foobaz') self.parser.add_argument('--fooble', action='store_true') + self.parser.add_argument('--foogle') self.assertRaisesRegex(argparse.ArgumentError, - "ambiguous option: --foob could match --foobaz, --fooble", - self.parser.parse_args, ['--foob']) + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob', '1', '--foogle', '2']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1', '--foogle', '2']) def test_os_error(self): self.parser.add_argument('file') @@ -6495,6 +6783,99 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(res.out.splitlines()[0].decode(), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. RFile.seen = {} diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index f052822cb45273..01d2e392302e86 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -789,6 +789,13 @@ def test_repr(self) -> None: with self.subTest(test_input=test): self.assertEqual(repr(ast.parse(test)), snapshot) + def test_repr_large_input_crash(self): + # gh-125010: Fix use-after-free in ast repr() + source = "0x0" + "e" * 10_000 + with self.assertRaisesRegex(ValueError, + r"Exceeds the limit \(\d+ digits\)"): + repr(ast.Constant(value=eval(source))) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 0777f39b572486..31d2a00dbb8c9c 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -213,6 +213,52 @@ async def run(): self.run_coro(run()) + def test_staggered_race_with_eager_tasks(self): + # See https://github.com/python/cpython/issues/124309 + + async def fail(): + await asyncio.sleep(0) + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: asyncio.sleep(2, result="sleep2"), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: fail() + ], + delay=0.25 + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIsInstance(excs[2], ValueError) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks_no_delay(self): + # See https://github.com/python/cpython/issues/124309 + async def fail(): + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: fail(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: asyncio.sleep(0, result="sleep0"), + ], + delay=None + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], ValueError) + self.assertEqual(len(excs), 2) + + self.run_coro(run()) + class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = tasks._PyTask diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 458b70451a306a..c566b28adb2408 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -659,6 +659,28 @@ def __del__(self): fut = self._new_future(loop=self.loop) fut.set_result(Evil()) + def test_future_cancelled_result_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.result() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + def test_future_cancelled_exception_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.exception() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 266f057f0776c3..45f70d09a2083a 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -93,8 +93,8 @@ async def main(): def test_asyncio_run_only_coro(self): for o in {1, lambda: None}: with self.subTest(obj=o), \ - self.assertRaisesRegex(ValueError, - 'a coroutine was expected'): + self.assertRaisesRegex(TypeError, + 'an awaitable is required'): asyncio.run(o) def test_asyncio_run_debug(self): @@ -319,19 +319,28 @@ async def f(): def test_run_non_coro(self): with asyncio.Runner() as runner: with self.assertRaisesRegex( - ValueError, - "a coroutine was expected" + TypeError, + "an awaitable is required" ): runner.run(123) def test_run_future(self): with asyncio.Runner() as runner: - with self.assertRaisesRegex( - ValueError, - "a coroutine was expected" - ): - fut = runner.get_loop().create_future() - runner.run(fut) + fut = runner.get_loop().create_future() + fut.set_result('done') + self.assertEqual('done', runner.run(fut)) + + def test_run_awaitable(self): + class MyAwaitable: + def __await__(self): + return self.run().__await__() + + @staticmethod + async def run(): + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(MyAwaitable())) def test_explicit_close(self): runner = asyncio.Runner() diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index e6e32f7dbbbcba..74941f704c4890 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -95,3 +95,30 @@ async def coro(index): self.assertEqual(len(excs), 2) self.assertIsInstance(excs[0], ValueError) self.assertIsInstance(excs[1], ValueError) + + + async def test_multiple_winners(self): + event = asyncio.Event() + + async def coro(index): + await event.wait() + return index + + async def do_set(): + event.set() + await asyncio.Event().wait() + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + do_set, + ], + delay=0.1, + ) + self.assertIs(winner, 0) + self.assertIs(index, 0) + self.assertEqual(len(excs), 3) + self.assertIsNone(excs[0], None) + self.assertIsInstance(excs[1], asyncio.CancelledError) + self.assertIsInstance(excs[2], asyncio.CancelledError) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index d32b7ff251885d..dbe5646c2b7c08 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1200,6 +1200,24 @@ async def handle_echo(reader, writer): messages = self._basetest_unhandled_exceptions(handle_echo) self.assertEqual(messages, []) + def test_open_connection_happy_eyeball_refcycles(self): + port = socket_helper.find_unused_port() + async def main(): + exc = None + try: + await asyncio.open_connection( + host="localhost", + port=port, + happy_eyeballs_delay=0.25, + ) + except* OSError as excs: + # can't use assertRaises because that clears frames + exc = excs.exceptions[0] + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), [main_coro]) + main_coro = main() + asyncio.run(main_coro) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 4852536defc93d..1b4de96a572fb9 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,7 +1,8 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. - +import sys +import gc import asyncio import contextvars import contextlib @@ -11,7 +12,6 @@ from test.test_asyncio.utils import await_without_task - # To prevent a warning "test altered the execution environment" def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -29,6 +29,15 @@ def get_error_types(eg): return {type(exc) for exc in eg.exceptions} +def no_other_refs(): + # due to gh-124392 coroutines now refer to their locals + coro = asyncio.current_task().get_coro() + frame = sys._getframe(1) + while coro.cr_frame != frame: + coro = coro.cr_await + return [coro] + + class TestTaskGroup(unittest.IsolatedAsyncioTestCase): async def test_taskgroup_01(self): @@ -899,6 +908,95 @@ async def outer(): await outer() + async def test_exception_refcycles_direct(self): + """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except ExceptionGroup as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_errors(self): + """Test that TaskGroup deletes self._errors, and __aexit__ args""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except* _Done as excs: + exc = excs.exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task(self): + """Test that TaskGroup deletes self._parent_task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + try: + async with asyncio.TaskGroup() as tg2: + tg2.create_task(coro_fn()) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_propagate_cancellation_error(self): + """Test that TaskGroup deletes propagate_cancellation_error""" + tg = asyncio.TaskGroup() + exc = None + + try: + async with asyncio.timeout(-1): + async with tg: + await asyncio.sleep(0) + except TimeoutError as e: + exc = e.__cause__ + + self.assertIsInstance(exc, asyncio.CancelledError) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_base_error(self): + """Test that TaskGroup deletes self._base_error""" + class MyKeyboardInterrupt(KeyboardInterrupt): + pass + + tg = asyncio.TaskGroup() + exc = None + + try: + async with tg: + raise MyKeyboardInterrupt + except MyKeyboardInterrupt as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 35893ab3118e1e..b8dbe7feaac3f4 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -15,6 +15,7 @@ import unittest import weakref import warnings +from ast import literal_eval from unittest import mock from http.server import HTTPServer @@ -56,24 +57,8 @@ def data_file(*filename): ONLYKEY = data_file('certdata', 'ssl_key.pem') SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem') SIGNING_CA = data_file('certdata', 'pycacert.pem') -PEERCERT = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} - +with open(data_file('certdata', 'keycert3.pem.reference')) as file: + PEERCERT = literal_eval(file.read()) def simple_server_sslcontext(): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 10c58c04dfd25e..f15dae13eb384e 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -1217,6 +1217,19 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) + def test_next_to_botframe(self): + # gh-125422 + # Check that next command won't go to the bottom frame. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, ''), ('step', ), + ('return', 2, ''), ('next', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '', 'exec')) + class TestRegressions(unittest.TestCase): def test_format_stack_entry_no_lineno(self): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 2ea97e797a4892..eb5906f8944c8e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2574,9 +2574,9 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = (1 << 30) - 1 + IMMORTAL_REFCOUNT = 3 << 29 else: - IMMORTAL_REFCOUNT = (1 << 32) - 1 + IMMORTAL_REFCOUNT = 3 << 30 IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) @@ -2607,6 +2607,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2685,6 +2686,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 1f9ffc5e9a5c33..f119d89c0ec39a 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -985,7 +985,7 @@ def assertFailure(self, *args): def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index bd521a509d07ec..85491a89947318 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -1,13 +1,20 @@ -import unittest +import codecs +import contextlib +import io +import re import sys +import unittest +import unittest.mock as mock +import _testcapi from test.support import import_helper _testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None +BAD_ARGUMENT = re.escape('bad argument type for built-in operation') -class CAPITest(unittest.TestCase): +class CAPIUnicodeTest(unittest.TestCase): # TODO: Test the following functions: # # PyUnicode_BuildEncodingMap @@ -516,5 +523,291 @@ def test_asrawunicodeescapestring(self): # CRASHES asrawunicodeescapestring(NULL) +class CAPICodecs(unittest.TestCase): + + def setUp(self): + # Encoding names are normalized internally by converting them + # to lowercase and their hyphens are replaced by underscores. + self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' + # Make sure that our custom codec is not already registered (that + # way we know whether we correctly unregistered the custom codec + # after a test or not). + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + # create the search function without registering yet + self._create_custom_codec() + + def _create_custom_codec(self): + def codec_encoder(m, errors='strict'): + return (type(m)().join(reversed(m)), len(m)) + + def codec_decoder(c, errors='strict'): + return (type(c)().join(reversed(c)), len(c)) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codec_encoder(input) + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codec_decoder(input) + + class StreamReader(codecs.StreamReader): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + info = codecs.CodecInfo( + encode=codec_encoder, + decode=codec_decoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + name=self.encoding_name + ) + + def search_function(encoding): + if encoding == self.encoding_name: + return info + return None + + self.codec_info = info + self.search_function = search_function + + @contextlib.contextmanager + def use_custom_encoder(self): + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + codecs.register(self.search_function) + yield + codecs.unregister(self.search_function) + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + + def test_codec_register(self): + search_function, encoding = self.search_function, self.encoding_name + # register the search function using the C API + self.assertIsNone(_testcapi.codec_register(search_function)) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIs(codecs.lookup(encoding), search_function(encoding)) + self.assertEqual(codecs.encode('123', encoding=encoding), '321') + # unregister the search function using the regular API + codecs.unregister(search_function) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_unregister(self): + search_function, encoding = self.search_function, self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding) + # register the search function using the regular API + codecs.register(search_function) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIsNotNone(codecs.lookup(encoding)) + # unregister the search function using the C API + self.assertIsNone(_testcapi.codec_unregister(search_function)) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_known_encoding(self): + self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') + self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) + self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) + self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) + + encoding_name = self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding_name) + + codecs.register(self.search_function) + self.addCleanup(codecs.unregister, self.search_function) + + for name in [ + encoding_name, + encoding_name.upper(), + encoding_name.replace('_', '-'), + ]: + with self.subTest(name): + self.assertTrue(_testcapi.codec_known_encoding(name)) + + def test_codec_encode(self): + encode = _testcapi.codec_encode + self.assertEqual(encode('a', 'utf-8', NULL), b'a') + self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') + self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') + + self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + encode('a', NULL, 'strict') + + def test_codec_decode(self): + decode = _testcapi.codec_decode + + s = 'a\xa1\u4f60\U0001f600' + b = s.encode() + + self.assertEqual(decode(b, 'utf-8', 'strict'), s) + self.assertEqual(decode(b, 'utf-8', NULL), s) + self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) + self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) + + # _codecs.decode() only reports an unknown error handling name when + # the corresponding error handling function is used; this difers + # from PyUnicode_Decode() which checks that both the encoding and + # the error handling name are recognized before even attempting to + # call the decoder. + self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') + self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') + + self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + decode(b, NULL, 'strict') + + def test_codec_encoder(self): + codec_encoder = _testcapi.codec_encoder + + with self.use_custom_encoder(): + encoder = codec_encoder(self.encoding_name) + self.assertIs(encoder, self.codec_info.encode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_encoder(NULL) + + def test_codec_decoder(self): + codec_decoder = _testcapi.codec_decoder + + with self.use_custom_encoder(): + decoder = codec_decoder(self.encoding_name) + self.assertIs(decoder, self.codec_info.decode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_decoder(NULL) + + def test_codec_incremental_encoder(self): + codec_incremental_encoder = _testcapi.codec_incremental_encoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + encoder = codec_incremental_encoder(encoding, errors) + self.assertIsInstance(encoder, self.codec_info.incrementalencoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_encoder(NULL, 'strict') + + def test_codec_incremental_decoder(self): + codec_incremental_decoder = _testcapi.codec_incremental_decoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + decoder = codec_incremental_decoder(encoding, errors) + self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_decoder(NULL, 'strict') + + def test_codec_stream_reader(self): + codec_stream_reader = _testcapi.codec_stream_reader + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_reader(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamreader) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_reader(NULL, stream, 'strict') + + def test_codec_stream_writer(self): + codec_stream_writer = _testcapi.codec_stream_writer + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_writer(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamwriter) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_writer(NULL, stream, 'strict') + + +class CAPICodecErrors(unittest.TestCase): + + def test_codec_register_error(self): + # for cleaning up between tests + from _codecs import _unregister_error as _codecs_unregister_error + + self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') + + def custom_error_handler(exc): + raise exc + + error_handler = mock.Mock(wraps=custom_error_handler) + _testcapi.codec_register_error('custom', error_handler) + self.addCleanup(_codecs_unregister_error, 'custom') + + self.assertRaises(UnicodeEncodeError, codecs.encode, + '\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + error_handler.reset_mock() + + self.assertRaises(UnicodeDecodeError, codecs.decode, + b'\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + + # _codecs._unregister_error directly delegates to the internal C + # function so a Python-level function test is sufficient (it is + # tested in test_codeccallbacks). + + def test_codec_lookup_error(self): + codec_lookup_error = _testcapi.codec_lookup_error + self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) + self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) + self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) + self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) + self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) + self.assertRaises(LookupError, codec_lookup_error, 'unknown') + + def test_codec_error_handlers(self): + exceptions = [ + # A UnicodeError with an empty message currently crashes: + # See: https://github.com/python/cpython/issues/123378 + # UnicodeEncodeError('bad', '', 0, 1, 'reason'), + UnicodeEncodeError('bad', 'x', 0, 1, 'reason'), + UnicodeEncodeError('bad', 'xyz123', 0, 1, 'reason'), + UnicodeEncodeError('bad', 'xyz123', 1, 4, 'reason'), + ] + + strict_handler = _testcapi.codec_strict_errors + for exc in exceptions: + with self.subTest(handler=strict_handler, exc=exc): + self.assertRaises(UnicodeEncodeError, strict_handler, exc) + + for handler in [ + _testcapi.codec_ignore_errors, + _testcapi.codec_replace_errors, + _testcapi.codec_xmlcharrefreplace_errors, + _testlimitedcapi.codec_namereplace_errors, + ]: + for exc in exceptions: + with self.subTest(handler=handler, exc=exc): + self.assertIsInstance(handler(exc), tuple) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 01637e1cb7b6e5..71fb9ae45c7c30 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -68,7 +68,7 @@ def test_config_get(self): ("parser_debug", bool, None), ("parse_argv", bool, None), ("pathconfig_warnings", bool, None), - ("perf_profiling", bool, None), + ("perf_profiling", int, None), ("platlibdir", str, "platlibdir"), ("prefix", str | None, "prefix"), ("program_name", str, None), diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index e6f85427214958..65d8242ad3fc60 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1903,6 +1903,39 @@ def test_recover_error(self): self.assertEqual(writer.finish(), 'Hello World.') + def test_unicode_equal(self): + unicode_equal = _testlimitedcapi.unicode_equal + + def copy(text): + return text.encode().decode() + + self.assertTrue(unicode_equal("", "")) + self.assertTrue(unicode_equal("abc", "abc")) + self.assertTrue(unicode_equal("abc", copy("abc"))) + self.assertTrue(unicode_equal("\u20ac", copy("\u20ac"))) + self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff"))) + + self.assertFalse(unicode_equal("abc", "abcd")) + self.assertFalse(unicode_equal("\u20ac", "\u20ad")) + self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe")) + + # str subclass + self.assertTrue(unicode_equal("abc", Str("abc"))) + self.assertTrue(unicode_equal(Str("abc"), "abc")) + self.assertFalse(unicode_equal("abc", Str("abcd"))) + self.assertFalse(unicode_equal(Str("abc"), "abcd")) + + # invalid type + for invalid_type in (b'bytes', 123, ("tuple",)): + with self.subTest(invalid_type=invalid_type): + with self.assertRaises(TypeError): + unicode_equal("abc", invalid_type) + with self.assertRaises(TypeError): + unicode_equal(invalid_type, "abc") + + # CRASHES unicode_equal("abc", NULL) + # CRASHES unicode_equal(NULL, "abc") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 902f788edc22f0..b88c4d16ba4ef4 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -932,6 +933,36 @@ class C: C.a = X() C.a = X() + @cpython_only + def test_detach_materialized_dict_no_memory(self): + # Skip test if _testcapi is not available: + import_helper.import_module('_testcapi') + + code = """if 1: + import test.support + import _testcapi + + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + assert ex.unraisable.exc_type is MemoryError + try: + d["a"] + except KeyError: + pass + else: + assert False, "KeyError not raised" + """ + rc, out, err = script_helper.assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertFalse(out, msg=out.decode('utf-8')) + self.assertFalse(err, msg=err.decode('utf-8')) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4991330489d139..86e5e5c1474674 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,3 +1,4 @@ +from _codecs import _unregister_error as _codecs_unregister_error import codecs import html.entities import itertools @@ -1210,7 +1211,6 @@ def replace_with_long(exc): '\ufffd\x00\x00' ) - def test_fake_error_class(self): handlers = [ codecs.strict_errors, @@ -1235,6 +1235,31 @@ class FakeUnicodeError(Exception): with self.assertRaises((TypeError, FakeUnicodeError)): handler(FakeUnicodeError()) + def test_reject_unregister_builtin_error_handler(self): + for name in [ + 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', + 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', + ]: + with self.subTest(name): + self.assertRaises(ValueError, _codecs_unregister_error, name) + + def test_unregister_custom_error_handler(self): + def custom_handler(exc): + raise exc + + custom_name = 'test.test_unregister_custom_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + codecs.register_error(custom_name, custom_handler) + self.assertIs(codecs.lookup_error(custom_name), custom_handler) + self.assertTrue(_codecs_unregister_error(custom_name)) + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + + def test_unregister_custom_unknown_error_handler(self): + unknown_name = 'test.test_unregister_custom_unknown_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + self.assertFalse(_codecs_unregister_error(unknown_name)) + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 428036e1765b8f..290656f070503a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2,7 +2,6 @@ import contextlib import copy import io -import locale import pickle import sys import unittest @@ -1812,16 +1811,10 @@ def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") + @support.run_with_locale('LC_CTYPE', 'tr_TR') def test_lookup_issue1813(self): # Issue #1813: under Turkish locales, lookup of some codecs failed # because 'I' is lowercased as "ı" (dotless i) - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') c = codecs.lookup('ASCII') self.assertEqual(c.name, 'ascii') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 736eff35c1d5f2..6f838da6018741 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import contextlib import dis import io +import itertools import math import opcode import os @@ -12,7 +13,10 @@ import types import textwrap import warnings -import _testinternalcapi +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test import support from test.support import (script_helper, requires_debug_ranges, run_code, @@ -475,6 +479,19 @@ def test_dead_code_with_except_handler_compiles(self): x = 2 """), '', 'exec') + def test_try_except_in_while_with_chained_condition_compiles(self): + # see gh-124871 + compile(textwrap.dedent(""" + name_1, name_2, name_3 = 1, 2, 3 + while name_3 <= name_2 > name_1: + try: + raise + except: + pass + finally: + pass + """), '', 'exec') + def test_compile_invalid_namedexpr(self): # gh-109351 m = ast.Module( @@ -1369,6 +1386,14 @@ def check_op_count(func, op, expected): actual += 1 self.assertEqual(actual, expected) + def check_consts(func, typ, expected): + slice_consts = 0 + consts = func.__code__.co_consts + for instr in dis.Bytecode(func): + if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): + slice_consts += 1 + self.assertEqual(slice_consts, expected) + def load(): return x[a:b] + x [a:] + x[:b] + x[:] @@ -1384,15 +1409,30 @@ def long_slice(): def aug(): x[a:b] += y - check_op_count(load, "BINARY_SLICE", 4) + def aug_const(): + x[1:2] += y + + def compound_const_slice(): + x[1:2:3, 4:5:6] = y + + check_op_count(load, "BINARY_SLICE", 3) check_op_count(load, "BUILD_SLICE", 0) - check_op_count(store, "STORE_SLICE", 4) + check_consts(load, slice, 1) + check_op_count(store, "STORE_SLICE", 3) check_op_count(store, "BUILD_SLICE", 0) + check_consts(store, slice, 1) check_op_count(long_slice, "BUILD_SLICE", 1) check_op_count(long_slice, "BINARY_SLICE", 0) check_op_count(aug, "BINARY_SLICE", 1) check_op_count(aug, "STORE_SLICE", 1) check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_consts(aug_const, slice, 1) + check_op_count(compound_const_slice, "BINARY_SLICE", 0) + check_op_count(compound_const_slice, "BUILD_SLICE", 0) + check_consts(compound_const_slice, slice, 0) + check_consts(compound_const_slice, tuple, 1) def test_compare_positions(self): for opname_prefix, op in [ @@ -1527,6 +1567,45 @@ async def name_4(): pass [[]] +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers @@ -2605,6 +2684,8 @@ def test_return_inside_async_with_block(self): """ self.check_stack_size(snippet, async_=True) +@support.cpython_only +@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class TestInstructionSequence(unittest.TestCase): def compare_instructions(self, seq, expected): self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], @@ -2648,6 +2729,22 @@ def test_nested(self): self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + def test_static_attributes_are_sorted(self): + code = ( + 'class T:\n' + ' def __init__(self):\n' + ' self.{V1} = 10\n' + ' self.{V2} = 10\n' + ' def foo(self):\n' + ' self.{V3} = 10\n' + ) + attributes = ("a", "b", "c") + for perm in itertools.permutations(attributes): + var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} + ns = run_code(code.format(**var_names)) + t = ns['T'] + self.assertEqual(t.__static_attributes__, attributes) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index 255be306156c0b..b06b9df9f5b0b8 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -60,6 +60,14 @@ def test_context_var_repr_1(self): c.reset(t) self.assertIn(' used ', repr(t)) + @isolated_context + def test_token_repr_1(self): + c = contextvars.ContextVar('a') + tok = c.set(1) + self.assertRegex(repr(tok), + r"^ at 0x[0-9a-fA-F]+>$") + def test_context_subclassing_1(self): with self.assertRaisesRegex(TypeError, 'not an acceptable base type'): class MyContextVar(contextvars.ContextVar): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index c718ee1203cbe0..ce5c03659f1979 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1103,6 +1103,12 @@ class mydialect(csv.Dialect): mydialect.quoting = None self.assertRaises(csv.Error, mydialect) + mydialect.quoting = 42 + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + 'bad "quoting" value') + mydialect.doublequote = True mydialect.quoting = csv.QUOTE_ALL mydialect.quotechar = '"' diff --git a/Lib/test/test_ctypes/test_structunion.py b/Lib/test/test_ctypes/test_structunion.py new file mode 100644 index 00000000000000..973ac3b2f1919d --- /dev/null +++ b/Lib/test/test_ctypes/test_structunion.py @@ -0,0 +1,353 @@ +"""Common tests for ctypes.Structure and ctypes.Union""" + +import unittest +from ctypes import (Structure, Union, POINTER, sizeof, alignment, + c_char, c_byte, c_ubyte, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ._support import (_CData, PyCStructType, UnionType, + Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) +from struct import calcsize + + +class StructUnionTestBase: + formats = {"c": c_char, + "b": c_byte, + "B": c_ubyte, + "h": c_short, + "H": c_ushort, + "i": c_int, + "I": c_uint, + "l": c_long, + "L": c_ulong, + "q": c_longlong, + "Q": c_ulonglong, + "f": c_float, + "d": c_double, + } + + def test_subclass(self): + class X(self.cls): + _fields_ = [("a", c_int)] + + class Y(X): + _fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_subclass_delayed(self): + class X(self.cls): + pass + self.assertEqual(sizeof(X), 0) + X._fields_ = [("a", c_int)] + + class Y(X): + pass + self.assertEqual(sizeof(Y), sizeof(X)) + Y._fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_inheritance_hierarchy(self): + self.assertEqual(self.cls.mro(), [self.cls, _CData, object]) + self.assertEqual(type(self.metacls), type) + + def test_type_flags(self): + for cls in self.cls, self.metacls: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewClass = self.metacls.__new__(self.metacls, 'NewClass', + (self.cls,), {}) + for cls in self.cls, NewClass: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(self.cls): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + self.metacls.__init__(T, 'ptr', (), {}) + + def test_alignment(self): + class X(self.cls): + _fields_ = [("x", c_char * 3)] + self.assertEqual(alignment(X), calcsize("s")) + self.assertEqual(sizeof(X), calcsize("3s")) + + class Y(self.cls): + _fields_ = [("x", c_char * 3), + ("y", c_int)] + self.assertEqual(alignment(Y), alignment(c_int)) + self.check_sizeof(Y, + struct_size=calcsize("3s i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class SI(self.cls): + _fields_ = [("a", X), + ("b", Y)] + self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) + self.check_sizeof(SI, + struct_size=calcsize("3s0i 3si 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class IS(self.cls): + _fields_ = [("b", Y), + ("a", X)] + + self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) + self.check_sizeof(IS, + struct_size=calcsize("3si 3s 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + self.assertEqual(alignment(XX), alignment(X)) + self.check_sizeof(XX, + struct_size=calcsize("3s 3s 0s"), + union_size=calcsize("3s")) + + def test_empty(self): + # I had problems with these + # + # Although these are pathological cases: Empty Structures! + class X(self.cls): + _fields_ = [] + + # Is this really the correct alignment, or should it be 0? + self.assertTrue(alignment(X) == 1) + self.assertTrue(sizeof(X) == 0) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + + self.assertEqual(alignment(XX), 1) + self.assertEqual(sizeof(XX), 0) + + def test_fields(self): + # test the offset and size attributes of Structure/Union fields. + class X(self.cls): + _fields_ = [("x", c_int), + ("y", c_char)] + + self.assertEqual(X.x.offset, 0) + self.assertEqual(X.x.size, sizeof(c_int)) + + if self.cls == Structure: + self.assertEqual(X.y.offset, sizeof(c_int)) + else: + self.assertEqual(X.y.offset, 0) + self.assertEqual(X.y.size, sizeof(c_char)) + + # readonly + self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) + self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) + + # XXX Should we check nested data types also? + # offset is always relative to the class... + + def test_invalid_field_types(self): + class POINT(self.cls): + pass + self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) + + def test_invalid_name(self): + # field name must be string + def declare_with_name(name): + class S(self.cls): + _fields_ = [(name, c_int)] + + self.assertRaises(TypeError, declare_with_name, b"x") + + def test_intarray_fields(self): + class SomeInts(self.cls): + _fields_ = [("a", c_int * 4)] + + # can use tuple to initialize array (but not list!) + self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) + self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) + self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) + self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) + # too long + # XXX Should raise ValueError?, not RuntimeError + self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) + + def test_huge_field_name(self): + # issue12881: segfault with large structure field names + def create_class(length): + class S(self.cls): + _fields_ = [('x' * length, c_int)] + + for length in [10 ** i for i in range(0, 8)]: + try: + create_class(length) + except MemoryError: + # MemoryErrors are OK, we just don't want to segfault + pass + + def test_abstract_class(self): + class X(self.cls): + _abstract_ = "something" + with self.assertRaisesRegex(TypeError, r"^abstract class$"): + X() + + def test_methods(self): + self.assertIn("in_dll", dir(type(self.cls))) + self.assertIn("from_address", dir(type(self.cls))) + self.assertIn("in_dll", dir(type(self.cls))) + + +class StructureTestCase(unittest.TestCase, StructUnionTestBase): + cls = Structure + metacls = PyCStructType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "PyCStructType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), struct_size) + + def test_simple_structs(self): + for code, tp in self.formats.items(): + class X(Structure): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("c%c0%c" % (code, code)), code)) + + +class UnionTestCase(unittest.TestCase, StructUnionTestBase): + cls = Union + metacls = UnionType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "UnionType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), union_size) + + def test_simple_unions(self): + for code, tp in self.formats.items(): + class X(Union): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("%c" % (code)), code)) + + +class PointerMemberTestBase: + def test(self): + # a Structure/Union with a POINTER field + class S(self.cls): + _fields_ = [("array", POINTER(c_int))] + + s = S() + # We can assign arrays of the correct type + s.array = (c_int * 3)(1, 2, 3) + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + + s.array[0] = 42 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [42, 2, 3]) + + s.array[0] = 1 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + +class PointerMemberTestCase_Struct(unittest.TestCase, PointerMemberTestBase): + cls = Structure + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertEqual(s.x, 12345678) + +class PointerMemberTestCase_Union(unittest.TestCase, PointerMemberTestBase): + cls = Union + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertFalse(s.p) # NULL pointers are falsy + + +class TestRecursiveBase: + def test_contains_itself(self): + class Recursive(self.cls): + pass + + try: + Recursive._fields_ = [("next", Recursive)] + except AttributeError as details: + self.assertIn("Structure or union cannot contain itself", + str(details)) + else: + self.fail("Structure or union cannot contain itself") + + + def test_vice_versa(self): + class First(self.cls): + pass + class Second(self.cls): + pass + + First._fields_ = [("second", Second)] + + try: + Second._fields_ = [("first", First)] + except AttributeError as details: + self.assertIn("_fields_ is final", str(details)) + else: + self.fail("AttributeError not raised") + +class TestRecursiveStructure(unittest.TestCase, TestRecursiveBase): + cls = Structure + +class TestRecursiveUnion(unittest.TestCase, TestRecursiveBase): + cls = Union diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 6cc09c8f2b5b59..0ec238e04b74cd 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,209 +1,24 @@ +"""Tests for ctypes.Structure + +Features common with Union should go in test_structunion.py instead. +""" + from platform import architecture as _architecture import struct import sys import unittest -from ctypes import (CDLL, Structure, Union, POINTER, sizeof, byref, alignment, +from ctypes import (CDLL, Structure, Union, POINTER, sizeof, byref, c_void_p, c_char, c_wchar, c_byte, c_ubyte, - c_uint8, c_uint16, c_uint32, - c_short, c_ushort, c_int, c_uint, - c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) + c_uint8, c_uint16, c_uint32, c_int, c_uint, + c_long, c_ulong, c_longlong, c_float, c_double) from ctypes.util import find_library -from struct import calcsize from collections import namedtuple from test import support from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") -from ._support import (_CData, PyCStructType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) - - -class SubclassesTest(unittest.TestCase): - def test_subclass(self): - class X(Structure): - _fields_ = [("a", c_int)] - - class Y(X): - _fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) - - def test_subclass_delayed(self): - class X(Structure): - pass - self.assertEqual(sizeof(X), 0) - X._fields_ = [("a", c_int)] - - class Y(X): - pass - self.assertEqual(sizeof(Y), sizeof(X)) - Y._fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) class StructureTestCase(unittest.TestCase): - formats = {"c": c_char, - "b": c_byte, - "B": c_ubyte, - "h": c_short, - "H": c_ushort, - "i": c_int, - "I": c_uint, - "l": c_long, - "L": c_ulong, - "q": c_longlong, - "Q": c_ulonglong, - "f": c_float, - "d": c_double, - } - - def test_inheritance_hierarchy(self): - self.assertEqual(Structure.mro(), [Structure, _CData, object]) - - self.assertEqual(PyCStructType.__name__, "PyCStructType") - self.assertEqual(type(PyCStructType), type) - - - def test_type_flags(self): - for cls in Structure, PyCStructType: - with self.subTest(cls=cls): - self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) - self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) - - def test_metaclass_details(self): - # Abstract classes (whose metaclass __init__ was not called) can't be - # instantiated directly - NewStructure = PyCStructType.__new__(PyCStructType, 'NewStructure', - (Structure,), {}) - for cls in Structure, NewStructure: - with self.subTest(cls=cls): - with self.assertRaisesRegex(TypeError, "abstract class"): - obj = cls() - - # Cannot call the metaclass __init__ more than once - class T(Structure): - _fields_ = [("x", c_char), - ("y", c_char)] - with self.assertRaisesRegex(SystemError, "already initialized"): - PyCStructType.__init__(T, 'ptr', (), {}) - - def test_simple_structs(self): - for code, tp in self.formats.items(): - class X(Structure): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("c%c0%c" % (code, code)), code)) - - def test_unions(self): - for code, tp in self.formats.items(): - class X(Union): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("%c" % (code)), code)) - - def test_struct_alignment(self): - class X(Structure): - _fields_ = [("x", c_char * 3)] - self.assertEqual(alignment(X), calcsize("s")) - self.assertEqual(sizeof(X), calcsize("3s")) - - class Y(Structure): - _fields_ = [("x", c_char * 3), - ("y", c_int)] - self.assertEqual(alignment(Y), alignment(c_int)) - self.assertEqual(sizeof(Y), calcsize("3si")) - - class SI(Structure): - _fields_ = [("a", X), - ("b", Y)] - self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) - self.assertEqual(sizeof(SI), calcsize("3s0i 3si 0i")) - - class IS(Structure): - _fields_ = [("b", Y), - ("a", X)] - - self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) - self.assertEqual(sizeof(IS), calcsize("3si 3s 0i")) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - self.assertEqual(alignment(XX), alignment(X)) - self.assertEqual(sizeof(XX), calcsize("3s 3s 0s")) - - def test_empty(self): - # I had problems with these - # - # Although these are pathological cases: Empty Structures! - class X(Structure): - _fields_ = [] - - class Y(Union): - _fields_ = [] - - # Is this really the correct alignment, or should it be 0? - self.assertTrue(alignment(X) == alignment(Y) == 1) - self.assertTrue(sizeof(X) == sizeof(Y) == 0) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - - self.assertEqual(alignment(XX), 1) - self.assertEqual(sizeof(XX), 0) - - def test_fields(self): - # test the offset and size attributes of Structure/Union fields. - class X(Structure): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, sizeof(c_int)) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - class X(Union): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, 0) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - # XXX Should we check nested data types also? - # offset is always relative to the class... - def test_packed(self): class X(Structure): _fields_ = [("a", c_byte), @@ -290,36 +105,6 @@ class POINT(Structure): pt = POINT(y=2, x=1) self.assertEqual((pt.x, pt.y), (1, 2)) - def test_invalid_field_types(self): - class POINT(Structure): - pass - self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) - - def test_invalid_name(self): - # field name must be string - def declare_with_name(name): - class S(Structure): - _fields_ = [(name, c_int)] - - self.assertRaises(TypeError, declare_with_name, b"x") - - def test_intarray_fields(self): - class SomeInts(Structure): - _fields_ = [("a", c_int * 4)] - - # can use tuple to initialize array (but not list!) - self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) - self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) - self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) - self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) - # too long - # XXX Should raise ValueError?, not RuntimeError - self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) - def test_nested_initializers(self): # test initializing nested structures class Phone(Structure): @@ -374,37 +159,12 @@ class Person(Structure): self.assertEqual(msg, "(Phone) TypeError: too many initializers") - def test_huge_field_name(self): - # issue12881: segfault with large structure field names - def create_class(length): - class S(Structure): - _fields_ = [('x' * length, c_int)] - - for length in [10 ** i for i in range(0, 8)]: - try: - create_class(length) - except MemoryError: - # MemoryErrors are OK, we just don't want to segfault - pass - def get_except(self, func, *args): try: func(*args) except Exception as detail: return detail.__class__, str(detail) - def test_abstract_class(self): - class X(Structure): - _abstract_ = "something" - # try 'X()' - cls, msg = self.get_except(eval, "X()", locals()) - self.assertEqual((cls, msg), (TypeError, "abstract class")) - - def test_methods(self): - self.assertIn("in_dll", dir(type(Structure))) - self.assertIn("from_address", dir(type(Structure))) - self.assertIn("in_dll", dir(type(Structure))) - def test_positional_args(self): # see also http://bugs.python.org/issue5042 class W(Structure): @@ -507,6 +267,8 @@ class X(Structure): self.assertEqual(s.second, got.second) def _test_issue18060(self, Vector): + # Regression tests for gh-62260 + # The call to atan2() should succeed if the # class fields were correctly cloned in the # subclasses. Otherwise, it will segfault. @@ -698,6 +460,7 @@ class Test3E(Structure): self.assertEqual(result.data[i], float(i+1)) def test_38368(self): + # Regression test for gh-82549 class U(Union): _fields_ = [ ('f1', c_uint8 * 16), @@ -719,9 +482,9 @@ class U(Union): self.assertEqual(f2, [0x4567, 0x0123, 0xcdef, 0x89ab, 0x3210, 0x7654, 0xba98, 0xfedc]) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_union_by_value(self): - # See bpo-16575 + # See gh-60779 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -800,9 +563,9 @@ class Test5(Structure): self.assertEqual(test5.nested.an_int, 0) self.assertEqual(test5.another_int, 0) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_bitfield_by_value(self): - # See bpo-16576 + # See gh-60780 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -882,75 +645,5 @@ class Test8(Union): 'a union by value, which is unsupported.') -class PointerMemberTestCase(unittest.TestCase): - - def test(self): - # a Structure with a POINTER field - class S(Structure): - _fields_ = [("array", POINTER(c_int))] - - s = S() - # We can assign arrays of the correct type - s.array = (c_int * 3)(1, 2, 3) - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - # The following are bugs, but are included here because the unittests - # also describe the current behaviour. - # - # This fails with SystemError: bad arg to internal function - # or with IndexError (with a patch I have) - - s.array[0] = 42 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [42, 2, 3]) - - s.array[0] = 1 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - def test_none_to_pointer_fields(self): - class S(Structure): - _fields_ = [("x", c_int), - ("p", POINTER(c_int))] - - s = S() - s.x = 12345678 - s.p = None - self.assertEqual(s.x, 12345678) - - -class TestRecursiveStructure(unittest.TestCase): - def test_contains_itself(self): - class Recursive(Structure): - pass - - try: - Recursive._fields_ = [("next", Recursive)] - except AttributeError as details: - self.assertIn("Structure or union cannot contain itself", - str(details)) - else: - self.fail("Structure or union cannot contain itself") - - - def test_vice_versa(self): - class First(Structure): - pass - class Second(Structure): - pass - - First._fields_ = [("second", Second)] - - try: - Second._fields_ = [("first", First)] - except AttributeError as details: - self.assertIn("_fields_ is final", str(details)) - else: - self.fail("AttributeError not raised") - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_unions.py b/Lib/test/test_ctypes/test_unions.py deleted file mode 100644 index e2dff0f22a9213..00000000000000 --- a/Lib/test/test_ctypes/test_unions.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest -from ctypes import Union, c_char -from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) - - -class ArrayTestCase(unittest.TestCase): - def test_inheritance_hierarchy(self): - self.assertEqual(Union.mro(), [Union, _CData, object]) - - self.assertEqual(UnionType.__name__, "UnionType") - self.assertEqual(type(UnionType), type) - - def test_type_flags(self): - for cls in Union, UnionType: - with self.subTest(cls=Union): - self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) - self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) - - def test_metaclass_details(self): - # Abstract classes (whose metaclass __init__ was not called) can't be - # instantiated directly - NewUnion = UnionType.__new__(UnionType, 'NewUnion', - (Union,), {}) - for cls in Union, NewUnion: - with self.subTest(cls=cls): - with self.assertRaisesRegex(TypeError, "abstract class"): - obj = cls() - - # Cannot call the metaclass __init__ more than once - class T(Union): - _fields_ = [("x", c_char), - ("y", c_char)] - with self.assertRaisesRegex(SystemError, "already initialized"): - UnionType.__init__(T, 'ptr', (), {}) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 83d10dd8579074..cc3aa561cd4c42 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -1081,6 +1081,14 @@ def test_resize_term(self): self.assertEqual(curses.LINES, lines) self.assertEqual(curses.COLS, cols) + with self.assertRaises(OverflowError): + curses.resize_term(35000, 1) + with self.assertRaises(OverflowError): + curses.resize_term(1, 35000) + # GH-120378: Overflow failure in resize_term() causes refresh to fail + tmp = curses.initscr() + tmp.erase() + @requires_curses_func('resizeterm') def test_resizeterm(self): curses.update_lines_cols() @@ -1095,6 +1103,14 @@ def test_resizeterm(self): self.assertEqual(curses.LINES, lines) self.assertEqual(curses.COLS, cols) + with self.assertRaises(OverflowError): + curses.resizeterm(35000, 1) + with self.assertRaises(OverflowError): + curses.resizeterm(1, 35000) + # GH-120378: Overflow failure in resizeterm() causes refresh to fail + tmp = curses.initscr() + tmp.erase() + def test_ungetch(self): curses.ungetch(b'A') self.assertEqual(self.stdscr.getkey(), 'A') diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 6934e88d9d338c..2e6c49e29ce828 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -17,7 +17,8 @@ from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict -from functools import total_ordering +from copy import deepcopy +from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. @@ -60,7 +61,7 @@ class C: x: int = field(default=1, default_factory=int) def test_field_repr(self): - int_field = field(default=1, init=True, repr=False) + int_field = field(default=1, init=True, repr=False, doc='Docstring') int_field.name = "id" repr_output = repr(int_field) expected_output = "Field(name='id',type=None," \ @@ -68,6 +69,7 @@ def test_field_repr(self): "init=True,repr=False,hash=None," \ "compare=True,metadata=mappingproxy({})," \ f"kw_only={MISSING!r}," \ + "doc='Docstring'," \ "_field_type=None)" self.assertEqual(repr_output, expected_output) @@ -3175,6 +3177,48 @@ class C: with self.assertRaisesRegex(TypeError, 'unhashable type'): hash(C({})) + def test_frozen_deepcopy_without_slots(self): + # see: https://github.com/python/cpython/issues/89683 + @dataclass(frozen=True, slots=False) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + def test_frozen_deepcopy_with_slots(self): + # see: https://github.com/python/cpython/issues/89683 + with self.subTest('generated __slots__'): + @dataclass(frozen=True, slots=True) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + with self.subTest('user-defined __slots__ and no __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + s: str + + # with user-defined slots, __getstate__ and __setstate__ are not + # automatically added, hence the error + err = r"^cannot\ assign\ to\ field\ 's'$" + self.assertRaisesRegex(FrozenInstanceError, err, deepcopy, C('')) + + with self.subTest('user-defined __slots__ and __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + __getstate__ = dataclasses._dataclass_getstate + __setstate__ = dataclasses._dataclass_setstate + + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + class TestSlots(unittest.TestCase): def test_simple(self): @@ -3261,7 +3305,7 @@ class Base(Root4): j: str h: str - self.assertEqual(Base.__slots__, ('y', )) + self.assertEqual(Base.__slots__, ('y',)) @dataclass(slots=True) class Derived(Base): @@ -3271,7 +3315,7 @@ class Derived(Base): k: str h: str - self.assertEqual(Derived.__slots__, ('z', )) + self.assertEqual(Derived.__slots__, ('z',)) @dataclass class AnotherDerived(Base): @@ -3279,6 +3323,24 @@ class AnotherDerived(Base): self.assertNotIn('__slots__', AnotherDerived.__dict__) + def test_slots_with_docs(self): + class Root: + __slots__ = {'x': 'x'} + + @dataclass(slots=True) + class Base(Root): + y1: int = field(doc='y1') + y2: int + + self.assertEqual(Base.__slots__, {'y1': 'y1', 'y2': None}) + + @dataclass(slots=True) + class Child(Base): + z1: int = field(doc='z1') + z2: int + + self.assertEqual(Child.__slots__, {'z1': 'z1', 'z2': None}) + def test_cant_inherit_from_iterator_slots(self): class Root: @@ -4255,6 +4317,23 @@ def test_funny_class_names_names(self): C = make_dataclass(classname, ['a', 'b']) self.assertEqual(C.__name__, classname) + def test_dataclass_decorator_default(self): + C = make_dataclass('C', [('x', int)], decorator=dataclass) + c = C(10) + self.assertEqual(c.x, 10) + + def test_dataclass_custom_decorator(self): + def custom_dataclass(cls, *args, **kwargs): + dc = dataclass(cls, *args, **kwargs) + dc.__custom__ = True + return dc + + C = make_dataclass('C', [('x', int)], decorator=custom_dataclass) + c = C(10) + self.assertEqual(c.x, 10) + self.assertEqual(c.__custom__, True) + + class TestReplace(unittest.TestCase): def test(self): @dataclass(frozen=True) @@ -4869,5 +4948,129 @@ class A: self.assertEqual(fs[0].name, 'x') +class TestZeroArgumentSuperWithSlots(unittest.TestCase): + def test_zero_argument_super(self): + @dataclass(slots=True) + class A: + def foo(self): + super() + + A().foo() + + def test_dunder_class_with_old_property(self): + @dataclass(slots=True) + class A: + def _get_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + return __class__ + + def _set_foo(slf, value): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + def _del_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + foo = property(_get_foo, _set_foo, _del_foo) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + def test_dunder_class_with_new_property(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return slf.__class__ + + @foo.setter + def foo(slf, value): + self.assertIs(__class__, type(slf)) + + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + # Test the parts of a property individually. + def test_slots_dunder_class_property_getter(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return __class__ + + a = A() + self.assertIs(a.foo, A) + + def test_slots_dunder_class_property_setter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.setter + def foo(slf, val): + self.assertIs(__class__, type(slf)) + + a = A() + a.foo = 4 + + def test_slots_dunder_class_property_deleter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + del a.foo + + def test_wrapped(self): + def mydecorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + + @dataclass(slots=True) + class A: + @mydecorator + def foo(self): + super() + + A().foo() + + def test_remembered_class(self): + # Apply the dataclass decorator manually (not when the class + # is created), so that we can keep a reference to the + # undecorated class. + class A: + def cls(self): + return __class__ + + self.assertIs(A().cls(), A) + + B = dataclass(slots=True)(A) + self.assertIs(B().cls(), B) + + # This is undesirable behavior, but is a function of how + # modifying __class__ in the closure works. I'm not sure this + # should be tested or not: I don't really want to guarantee + # this behavior, but I don't want to lose the point that this + # is how it works. + + # The underlying class is "broken" by changing its __class__ + # in A.foo() to B. This normally isn't a problem, because no + # one will be keeping a reference to the underlying class A. + self.assertIs(A().cls(), B) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 12479e32d0f5db..bc6c6427740949 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -812,6 +812,29 @@ def test_explicit_context_create_from_float(self): x = random.expovariate(0.01) * (random.random() * 2.0 - 1.0) self.assertEqual(x, float(nc.create_decimal(x))) # roundtrip + def test_from_number(self, cls=None): + Decimal = self.decimal.Decimal + if cls is None: + cls = Decimal + + def check(arg, expected): + d = cls.from_number(arg) + self.assertIs(type(d), cls) + self.assertEqual(d, expected) + + check(314, Decimal(314)) + check(3.14, Decimal.from_float(3.14)) + check(Decimal('3.14'), Decimal('3.14')) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '314') + self.assertRaises(TypeError, cls.from_number, (0, (3, 1, 4), 0)) + self.assertRaises(TypeError, cls.from_number, object()) + + def test_from_number_subclass(self, cls=None): + class DecimalSubclass(self.decimal.Decimal): + pass + self.test_from_number(DecimalSubclass) + def test_unicode_digits(self): Decimal = self.decimal.Decimal @@ -1253,7 +1276,7 @@ def test_deprecated_N_format(self): self.assertRaises(ValueError, format, h, '10Nf') self.assertRaises(ValueError, format, h, 'Nx') - @run_with_locale('LC_ALL', 'ps_AF') + @run_with_locale('LC_ALL', 'ps_AF', '') def test_wide_char_separator_decimal_point(self): # locale with wide char separator and decimal point Decimal = self.decimal.Decimal @@ -4381,7 +4404,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): diff --git a/Lib/test/test_dict_version.py b/Lib/test/test_dict_version.py deleted file mode 100644 index 243084c75c42bc..00000000000000 --- a/Lib/test/test_dict_version.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Test implementation of the PEP 509: dictionary versioning. -""" -import unittest -from test.support import import_helper - -# PEP 509 is implemented in CPython but other Python implementations -# don't require to implement it -_testcapi = import_helper.import_module('_testcapi') - - -class DictVersionTests(unittest.TestCase): - type2test = dict - - def setUp(self): - self.seen_versions = set() - self.dict = None - - def check_version_unique(self, mydict): - version = _testcapi.dict_get_version(mydict) - self.assertNotIn(version, self.seen_versions) - self.seen_versions.add(version) - - def check_version_changed(self, mydict, method, *args, **kw): - result = method(*args, **kw) - self.check_version_unique(mydict) - return result - - def check_version_dont_change(self, mydict, method, *args, **kw): - version1 = _testcapi.dict_get_version(mydict) - self.seen_versions.add(version1) - - result = method(*args, **kw) - - version2 = _testcapi.dict_get_version(mydict) - self.assertEqual(version2, version1, "version changed") - - return result - - def new_dict(self, *args, **kw): - d = self.type2test(*args, **kw) - self.check_version_unique(d) - return d - - def test_constructor(self): - # new empty dictionaries must all have an unique version - empty1 = self.new_dict() - empty2 = self.new_dict() - empty3 = self.new_dict() - - # non-empty dictionaries must also have an unique version - nonempty1 = self.new_dict(x='x') - nonempty2 = self.new_dict(x='x', y='y') - - def test_copy(self): - d = self.new_dict(a=1, b=2) - - d2 = self.check_version_dont_change(d, d.copy) - - # dict.copy() must create a dictionary with a new unique version - self.check_version_unique(d2) - - def test_setitem(self): - d = self.new_dict() - - # creating new keys must change the version - self.check_version_changed(d, d.__setitem__, 'x', 'x') - self.check_version_changed(d, d.__setitem__, 'y', 'y') - - # changing values must change the version - self.check_version_changed(d, d.__setitem__, 'x', 1) - self.check_version_changed(d, d.__setitem__, 'y', 2) - - def test_setitem_same_value(self): - value = object() - d = self.new_dict() - - # setting a key must change the version - self.check_version_changed(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.__setitem__ - # must change the version - self.check_version_dont_change(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.update - # must change the version - self.check_version_dont_change(d, d.update, key=value) - - d2 = self.new_dict(key=value) - self.check_version_dont_change(d, d.update, d2) - - def test_setitem_equal(self): - class AlwaysEqual: - def __eq__(self, other): - return True - - value1 = AlwaysEqual() - value2 = AlwaysEqual() - self.assertTrue(value1 == value2) - self.assertFalse(value1 != value2) - self.assertIsNot(value1, value2) - - d = self.new_dict() - self.check_version_changed(d, d.__setitem__, 'key', value1) - self.assertIs(d['key'], value1) - - # setting a key to a value equal to the current value - # with dict.__setitem__() must change the version - self.check_version_changed(d, d.__setitem__, 'key', value2) - self.assertIs(d['key'], value2) - - # setting a key to a value equal to the current value - # with dict.update() must change the version - self.check_version_changed(d, d.update, key=value1) - self.assertIs(d['key'], value1) - - d2 = self.new_dict(key=value2) - self.check_version_changed(d, d.update, d2) - self.assertIs(d['key'], value2) - - def test_setdefault(self): - d = self.new_dict() - - # setting a key with dict.setdefault() must change the version - self.check_version_changed(d, d.setdefault, 'key', 'value1') - - # don't change the version if the key already exists - self.check_version_dont_change(d, d.setdefault, 'key', 'value2') - - def test_delitem(self): - d = self.new_dict(key='value') - - # deleting a key with dict.__delitem__() must change the version - self.check_version_changed(d, d.__delitem__, 'key') - - # don't change the version if the key doesn't exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.__delitem__, 'key') - - def test_pop(self): - d = self.new_dict(key='value') - - # pop() must change the version if the key exists - self.check_version_changed(d, d.pop, 'key') - - # pop() must not change the version if the key does not exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.pop, 'key') - - def test_popitem(self): - d = self.new_dict(key='value') - - # popitem() must change the version if the dict is not empty - self.check_version_changed(d, d.popitem) - - # popitem() must not change the version if the dict is empty - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.popitem) - - def test_update(self): - d = self.new_dict(key='value') - - # update() calling with no argument must not change the version - self.check_version_dont_change(d, d.update) - - # update() must change the version - self.check_version_changed(d, d.update, key='new value') - - d2 = self.new_dict(key='value 3') - self.check_version_changed(d, d.update, d2) - - def test_clear(self): - d = self.new_dict(key='value') - - # clear() must change the version if the dict is not empty - self.check_version_changed(d, d.clear) - - # clear() must not change the version if the dict is empty - self.check_version_dont_change(d, d.clear) - - -class Dict(dict): - pass - - -class DictSubtypeVersionTests(DictVersionTests): - type2test = Dict - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index bccd2182412577..1ee0fbe98914be 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -380,6 +380,23 @@ def wrap_func_w_kwargs(): RETURN_CONST 3 (None) """ +fn_with_annotate_str = """ +def foo(a: int, b: str) -> str: + return a * b +""" + +dis_fn_with_annotate_str = """\ + 0 RESUME 0 + + 2 LOAD_CONST 0 (", line 2>) + MAKE_FUNCTION + LOAD_CONST 1 (", line 2>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 16 (annotate) + STORE_NAME 0 (foo) + RETURN_CONST 2 (None) +""" + compound_stmt_str = """\ x = 0 while 1: @@ -1098,6 +1115,7 @@ def test_disassemble_str(self): self.do_disassembly_test(expr_str, dis_expr_str) self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) self.do_disassembly_test(annot_stmt_str, dis_annot_stmt_str) + self.do_disassembly_test(fn_with_annotate_str, dis_fn_with_annotate_str) self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) def test_disassemble_bytes(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7c5cb855a397ab..035d4418b15c51 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -168,7 +168,8 @@ def run_repeated_init_and_subinterpreters(self): # Parse the line from the loop. The first line is the main # interpreter and the 3 afterward are subinterpreters. interp = Interp(*match.groups()) - if support.verbose > 1: + if support.verbose > 2: + # 5 lines per pass is super-spammy, so limit that to -vvv print(interp) self.assertTrue(interp.interp) self.assertTrue(interp.tstate) @@ -279,6 +280,10 @@ def test_pre_initialization_api(self): """ env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env) + if support.verbose > 1: + print() + print(out) + print(err) if MS_WINDOWS: expected_path = self.test_exe else: @@ -296,6 +301,10 @@ def test_pre_initialization_sys_options(self): env['PYTHONPATH'] = os.pathsep.join(sys.path) out, err = self.run_embedded_interpreter( "test_pre_initialization_sys_options", env=env) + if support.verbose > 1: + print() + print(out) + print(err) expected_output = ( "sys.warnoptions: ['once', 'module', 'default']\n" "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" @@ -560,7 +569,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'cpu_count': -1, 'faulthandler': False, 'tracemalloc': 0, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': False, 'code_debug_ranges': True, 'show_ref_count': False, @@ -652,7 +661,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): use_hash_seed=False, faulthandler=False, tracemalloc=False, - perf_profiling=False, + perf_profiling=0, pathconfig_warnings=False, ) if MS_WINDOWS: @@ -966,7 +975,7 @@ def test_init_from_config(self): 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'show_ref_count': True, @@ -1031,7 +1040,7 @@ def test_init_compat_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1051,6 +1060,7 @@ def test_init_compat_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = 1 @@ -1066,7 +1076,7 @@ def test_init_python_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1086,6 +1096,7 @@ def test_init_python_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = True @@ -1763,6 +1774,7 @@ def test_initconfig_api(self): 'xoptions': {'faulthandler': True}, 'hash_seed': 10, 'use_hash_seed': True, + 'perf_profiling': 2, } config_dev_mode(preconfig, config) self.check_all_configs("test_initconfig_api", config, preconfig, diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index ba858c49400911..b3c21cd4f3d585 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -8,6 +8,7 @@ import weakref import errno from codecs import BOM_UTF8 +from itertools import product from textwrap import dedent from test.support import (captured_stderr, check_impl_detail, @@ -1336,6 +1337,29 @@ def test_unicode_errors_no_object(self): for klass in klasses: self.assertEqual(str(klass.__new__(klass)), "") + def test_unicode_error_str_does_not_crash(self): + # Test that str(UnicodeError(...)) does not crash. + # See https://github.com/python/cpython/issues/123378. + + for start, end, objlen in product( + range(-5, 5), + range(-5, 5), + range(7), + ): + obj = 'a' * objlen + with self.subTest('encode', objlen=objlen, start=start, end=end): + exc = UnicodeEncodeError('utf-8', obj, start, end, '') + self.assertIsInstance(str(exc), str) + + with self.subTest('translate', objlen=objlen, start=start, end=end): + exc = UnicodeTranslateError(obj, start, end, '') + self.assertIsInstance(str(exc), str) + + encoded = obj.encode() + with self.subTest('decode', objlen=objlen, start=start, end=end): + exc = UnicodeDecodeError('utf-8', encoded, start, end, '') + self.assertIsInstance(str(exc), str) + @no_tracing def test_badisinstance(self): # Bug #2542: if issubclass(e, MyException) raises an exception, diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 048bb14509064b..f588e16b70123a 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -175,7 +175,7 @@ def check(s): # non-UTF-8 byte string check(b'123\xa0') - @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') + @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE', '') def test_float_with_comma(self): # set locale to something that doesn't use '.' for the decimal point # float must not accept the locale specific decimal point but diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 4907f4093f52c9..98dccbec9566ac 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -283,6 +283,13 @@ def __repr__(self): class RectComplex(Rect, complex): pass +class Ratio: + def __init__(self, ratio): + self._ratio = ratio + def as_integer_ratio(self): + return self._ratio + + class FractionTest(unittest.TestCase): def assertTypedEquals(self, expected, actual): @@ -355,14 +362,9 @@ def testInitFromDecimal(self): self.assertRaises(OverflowError, F, Decimal('-inf')) def testInitFromIntegerRatio(self): - class Ratio: - def __init__(self, ratio): - self._ratio = ratio - def as_integer_ratio(self): - return self._ratio - self.assertEqual((7, 3), _components(F(Ratio((7, 3))))) - errmsg = "argument should be a string or a number" + errmsg = (r"argument should be a string or a Rational instance or " + r"have the as_integer_ratio\(\) method") # the type also has an "as_integer_ratio" attribute. self.assertRaisesRegex(TypeError, errmsg, F, Ratio) # bad ratio @@ -388,6 +390,8 @@ class B(metaclass=M): pass self.assertRaisesRegex(TypeError, errmsg, F, B) self.assertRaisesRegex(TypeError, errmsg, F, B()) + self.assertRaises(TypeError, F.from_number, B) + self.assertRaises(TypeError, F.from_number, B()) def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) @@ -594,6 +598,37 @@ def testFromDecimal(self): ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) + def testFromNumber(self, cls=F): + def check(arg, numerator, denominator): + f = cls.from_number(arg) + self.assertIs(type(f), cls) + self.assertEqual(f.numerator, numerator) + self.assertEqual(f.denominator, denominator) + + check(10, 10, 1) + check(2.5, 5, 2) + check(Decimal('2.5'), 5, 2) + check(F(22, 7), 22, 7) + check(DummyFraction(22, 7), 22, 7) + check(Rat(22, 7), 22, 7) + check(Ratio((22, 7)), 22, 7) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '5/2') + self.assertRaises(TypeError, cls.from_number, []) + self.assertRaises(OverflowError, cls.from_number, float('inf')) + self.assertRaises(OverflowError, cls.from_number, Decimal('inf')) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + check(a, 9, 5) + + def testFromNumber_subclass(self): + self.testFromNumber(DummyFraction) + + def test_is_integer(self): self.assertTrue(F(1, 1).is_integer()) self.assertTrue(F(-1, 1).is_integer()) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index ca88e657367d9a..32de8ed9a13f80 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -494,6 +494,27 @@ class ObjectSubclass: with self.assertRaises(TypeError): proxy[obj] = 0 + def test_constructor(self): + FrameLocalsProxy = type([sys._getframe().f_locals + for x in range(1)][0]) + self.assertEqual(FrameLocalsProxy.__name__, 'FrameLocalsProxy') + + def make_frame(): + x = 1 + y = 2 + return sys._getframe() + + proxy = FrameLocalsProxy(make_frame()) + self.assertEqual(proxy, {'x': 1, 'y': 2}) + + # constructor expects 1 frame argument + with self.assertRaises(TypeError): + FrameLocalsProxy() # no arguments + with self.assertRaises(TypeError): + FrameLocalsProxy(123) # wrong type + with self.assertRaises(TypeError): + FrameLocalsProxy(frame=sys._getframe()) # no keyword arguments + class FrameLocalsProxyMappingTests(mapping_tests.TestHashMappingProtocol): """Test that FrameLocalsProxy behaves like a Mapping (with exceptions)""" diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 3126458e08e50a..80daf0d9cae9e0 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -142,41 +142,6 @@ def writer_func(l): for ref in thread_list: self.assertIsNone(ref()) - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_dict_version(self): - dict_version = _testcapi.dict_version - THREAD_COUNT = 10 - DICT_COUNT = 10000 - lists = [] - writers = [] - - def writer_func(thread_list): - for i in range(DICT_COUNT): - thread_list.append(dict_version({})) - - for x in range(THREAD_COUNT): - thread_list = [] - lists.append(thread_list) - writer = Thread(target=partial(writer_func, thread_list)) - writers.append(writer) - - for writer in writers: - writer.start() - - for writer in writers: - writer.join() - - total_len = 0 - values = set() - for thread_list in lists: - for v in thread_list: - if v in values: - print('dup', v, (v/4096)%256) - values.add(v) - total_len += len(thread_list) - versions = set(dict_version for thread_list in lists for dict_version in thread_list) - self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py index c6b58fcd86f449..a705161369e8dd 100644 --- a/Lib/test/test_free_threading/test_list.py +++ b/Lib/test/test_free_threading/test_list.py @@ -3,10 +3,13 @@ from threading import Thread from unittest import TestCase -from test import support from test.support import threading_helper +NTHREAD = 10 +OBJECT_COUNT = 5_000 + + class C: def __init__(self, v): self.v = v @@ -14,11 +17,8 @@ def __init__(self, v): @threading_helper.requires_working_threading() class TestList(TestCase): - @support.requires_resource('cpu') def test_racing_iter_append(self): - l = [] - OBJECT_COUNT = 10000 def writer_func(): for i in range(OBJECT_COUNT): @@ -34,7 +34,7 @@ def reader_func(): writer = Thread(target=writer_func) readers = [] - for x in range(30): + for x in range(NTHREAD): reader = Thread(target=reader_func) readers.append(reader) reader.start() @@ -44,39 +44,32 @@ def reader_func(): for reader in readers: reader.join() - @support.requires_resource('cpu') def test_racing_iter_extend(self): - iters = [ - lambda x: [x], - ] - for iter_case in iters: - with self.subTest(iter=iter_case): - l = [] - OBJECT_COUNT = 10000 - - def writer_func(): - for i in range(OBJECT_COUNT): - l.extend(iter_case(C(i + OBJECT_COUNT))) - - def reader_func(): - while True: - count = len(l) - for i, x in enumerate(l): - self.assertEqual(x.v, i + OBJECT_COUNT) - if count == OBJECT_COUNT: - break - - writer = Thread(target=writer_func) - readers = [] - for x in range(30): - reader = Thread(target=reader_func) - readers.append(reader) - reader.start() - - writer.start() - writer.join() - for reader in readers: - reader.join() + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend([C(i + OBJECT_COUNT)]) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() if __name__ == "__main__": diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index be582455d118ac..8fec01715531cb 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -7,7 +7,6 @@ import weakref from sys import monitoring -from test import support from test.support import threading_helper from threading import Thread, _PyRLock from unittest import TestCase @@ -15,7 +14,7 @@ class InstrumentationMultiThreadedMixin: thread_count = 10 - func_count = 200 + func_count = 50 fib = 12 def after_threads(self): @@ -37,14 +36,13 @@ def work(self, n, funcs): def start_work(self, n, funcs): # With the GIL builds we need to make sure that the hooks have # a chance to run as it's possible to run w/o releasing the GIL. - time.sleep(1) + time.sleep(0.1) self.work(n, funcs) def after_test(self): """Runs once after the test is done""" pass - @support.requires_resource('cpu') def test_instrumentation(self): # Setup a bunch of functions which will need instrumentation... funcs = [] @@ -220,29 +218,31 @@ def test_register_callback(self): for ref in self.refs: self.assertEqual(ref(), None) - @support.requires_resource('cpu') def test_set_local_trace_opcodes(self): def trace(frame, event, arg): frame.f_trace_opcodes = True return trace + loops = 1_000 + sys.settrace(trace) try: l = _PyRLock() def f(): - for i in range(3000): + for i in range(loops): with l: pass t = Thread(target=f) t.start() - for i in range(3000): + for i in range(loops): with l: pass t.join() finally: sys.settrace(None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 977bfd2c7fd2f7..51463b6bb8c1b4 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -5,7 +5,6 @@ from threading import Thread from unittest import TestCase -from test import support from test.support import threading_helper @@ -97,8 +96,9 @@ def reader_func(): self.run_one(writer_func, reader_func) - @support.requires_resource('cpu') def test___class___modification(self): + loops = 200 + class Foo: pass @@ -108,7 +108,7 @@ class Bar: thing = Foo() def work(): foo = thing - for _ in range(5000): + for _ in range(loops): foo.__class__ = Bar type(foo) foo.__class__ = Foo diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index b3fc5ad42e7fde..d919d62613ea7c 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -98,7 +98,12 @@ def test___globals__(self): (AttributeError, TypeError)) def test___builtins__(self): - self.assertIs(self.b.__builtins__, __builtins__) + if __name__ == "__main__": + builtins_dict = __builtins__.__dict__ + else: + builtins_dict = __builtins__ + + self.assertIs(self.b.__builtins__, builtins_dict) self.cannot_set_attr(self.b, '__builtins__', 2, (AttributeError, TypeError)) @@ -108,7 +113,7 @@ def func(s): return len(s) ns = {} func2 = type(func)(func.__code__, ns) self.assertIs(func2.__globals__, ns) - self.assertIs(func2.__builtins__, __builtins__) + self.assertIs(func2.__builtins__, builtins_dict) # Make sure that the function actually works. self.assertEqual(func2("abc"), 3) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 837f3795f0842d..bdaa9a7ec4f020 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -6,6 +6,7 @@ from itertools import permutations import pickle from random import choice +import re import sys from test import support import threading @@ -210,6 +211,51 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') + def test_placeholders_trailing_raise(self): + PH = self.module.Placeholder + for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: + with self.assertRaises(TypeError): + self.partial(capture, *args) + + def test_placeholders(self): + PH = self.module.Placeholder + # 1 Placeholder + args = (PH, 0) + p = self.partial(capture, *args) + actual_args, actual_kwds = p('x') + self.assertEqual(actual_args, ('x', 0)) + self.assertEqual(actual_kwds, {}) + # 2 Placeholders + args = (PH, 0, PH, 1) + p = self.partial(capture, *args) + with self.assertRaises(TypeError): + p('x') + actual_args, actual_kwds = p('x', 'y') + self.assertEqual(actual_args, ('x', 0, 'y', 1)) + self.assertEqual(actual_kwds, {}) + + def test_placeholders_optimization(self): + PH = self.module.Placeholder + p = self.partial(capture, PH, 0) + p2 = self.partial(p, PH, 1, 2, 3) + self.assertEqual(p2.args, (PH, 0, 1, 2, 3)) + p3 = self.partial(p2, -1, 4) + actual_args, actual_kwds = p3(5) + self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5)) + self.assertEqual(actual_kwds, {}) + # inner partial has placeholders and outer partial has no args case + p = self.partial(capture, PH, 0) + p2 = self.partial(p) + self.assertEqual(p2.args, (PH, 0)) + self.assertEqual(p2(1), ((1, 0), {})) + + def test_construct_placeholder_singleton(self): + PH = self.module.Placeholder + tp = type(PH) + self.assertIs(tp(), PH) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) @@ -311,6 +357,23 @@ def test_setstate(self): self.assertEqual(f(2), ((2,), {})) self.assertEqual(f(), ((), {})) + # Set State with placeholders + PH = self.module.Placeholder + f = self.partial(signature) + f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) + self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) + msg_regex = re.escape("missing positional arguments in 'partial' call; " + "expected at least 1, got 0") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f() + self.assertEqual(f(2), ((2, 1), dict(a=10))) + + # Trailing Placeholder error + f = self.partial(signature) + msg_regex = re.escape("trailing Placeholders are not allowed") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) @@ -456,6 +519,19 @@ def __str__(self): self.assertIn('astr', r) self.assertIn("['sth']", r) + def test_placeholders_refcount_smoke(self): + PH = self.module.Placeholder + # sum supports vector call + lst1, start = [], [] + sum_lists = self.partial(sum, PH, start) + for i in range(10): + sum_lists([lst1, lst1]) + # collections.ChainMap initializer does not support vectorcall + map1, map2 = {}, {} + partial_cm = self.partial(collections.ChainMap, PH, map1) + for i in range(10): + partial_cm(map2, map2) + class TestPartialPy(TestPartial, unittest.TestCase): module = py_functools @@ -480,6 +556,19 @@ class TestPartialCSubclass(TestPartialC): class TestPartialPySubclass(TestPartialPy): partial = PyPartialSubclass + def test_subclass_optimization(self): + # `partial` input to `partial` subclass + p = py_functools.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + # `partial` subclass input to `partial` subclass + p = self.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + + class TestPartialMethod(unittest.TestCase): class A(object): @@ -617,6 +706,20 @@ def f(a, b, /): p = functools.partial(f, 1) self.assertEqual(p(2), f(1, 2)) + def test_subclass_optimization(self): + class PartialMethodSubclass(functools.partialmethod): + pass + # `partialmethod` input to `partialmethod` subclass + p = functools.partialmethod(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + # `partialmethod` subclass input to `partialmethod` subclass + p = PartialMethodSubclass(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + class TestUpdateWrapper(unittest.TestCase): diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 906f9884d6792f..bb7df1f5cfa7f7 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1048,6 +1048,24 @@ class Z: callback.assert_not_called() gc.enable() + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + class IncrementalGCTests(unittest.TestCase): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 5d20e3c30bcf10..cd3718b80612bd 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -60,7 +60,7 @@ def test_effect_sizes(self): stack.pop(y) stack.pop(x) for out in outputs: - stack.push(Local.local(out)) + stack.push(Local.undefined(out)) self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") self.assertEqual(stack.top_offset.to_c(), "1 - oparg - oparg*2 + oparg*4") @@ -122,7 +122,7 @@ def run_cases_test(self, input: str, expected: str): def test_inst_no_args(self): input = """ inst(OP, (--)) { - spam(); + SPAM(); } """ output = """ @@ -130,7 +130,7 @@ def test_inst_no_args(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - spam(); + SPAM(); DISPATCH(); } """ @@ -139,7 +139,8 @@ def test_inst_no_args(self): def test_inst_one_pop(self): input = """ inst(OP, (value --)) { - spam(value); + SPAM(value); + DEAD(value); } """ output = """ @@ -149,7 +150,7 @@ def test_inst_one_pop(self): INSTRUCTION_STATS(OP); _PyStackRef value; value = stack_pointer[-1]; - spam(value); + SPAM(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -160,7 +161,7 @@ def test_inst_one_pop(self): def test_inst_one_push(self): input = """ inst(OP, (-- res)) { - res = spam(); + res = SPAM(); } """ output = """ @@ -169,7 +170,7 @@ def test_inst_one_push(self): next_instr += 1; INSTRUCTION_STATS(OP); _PyStackRef res; - res = spam(); + res = SPAM(); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -181,7 +182,8 @@ def test_inst_one_push(self): def test_inst_one_push_one_pop(self): input = """ inst(OP, (value -- res)) { - res = spam(value); + res = SPAM(value); + DEAD(value); } """ output = """ @@ -192,7 +194,7 @@ def test_inst_one_push_one_pop(self): _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; - res = spam(value); + res = SPAM(value); stack_pointer[-1] = res; DISPATCH(); } @@ -202,7 +204,9 @@ def test_inst_one_push_one_pop(self): def test_binary_op(self): input = """ inst(OP, (left, right -- res)) { - res = spam(left, right); + res = SPAM(left, right); + INPUTS_DEAD(); + } """ output = """ @@ -215,7 +219,7 @@ def test_binary_op(self): _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = spam(left, right); + res = SPAM(left, right); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -227,7 +231,8 @@ def test_binary_op(self): def test_overlap(self): input = """ inst(OP, (left, right -- left, result)) { - result = spam(left, right); + result = SPAM(left, right); + INPUTS_DEAD(); } """ output = """ @@ -240,7 +245,7 @@ def test_overlap(self): _PyStackRef result; right = stack_pointer[-1]; left = stack_pointer[-2]; - result = spam(left, right); + result = SPAM(left, right); stack_pointer[-1] = result; DISPATCH(); } @@ -249,7 +254,8 @@ def test_overlap(self): def test_predictions(self): input = """ - inst(OP1, (arg -- rest)) { + inst(OP1, (arg -- res)) { + res = Py_None; } inst(OP3, (arg -- res)) { DEOPT_IF(xxx); @@ -263,7 +269,9 @@ def test_predictions(self): next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED(OP1); - stack_pointer[-1] = rest; + _PyStackRef res; + res = Py_None; + stack_pointer[-1] = res; DISPATCH(); } @@ -281,6 +289,67 @@ def test_predictions(self): """ self.run_cases_test(input, output) + def test_sync_sp(self): + input = """ + inst(A, (arg -- res)) { + SYNC_SP(); + escaping_call(); + res = Py_None; + } + inst(B, (arg -- res)) { + res = Py_None; + SYNC_SP(); + escaping_call(); + } + """ + output = """ + TARGET(A) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(A); + _PyStackRef res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = Py_None; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(B) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(B); + _PyStackRef res; + res = Py_None; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_pep7_condition(self): + input = """ + inst(OP, (arg1 -- out)) { + if (arg1) + out = 0; + else { + out = 1; + } + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + def test_error_if_plain(self): input = """ inst(OP, (--)) { @@ -319,8 +388,10 @@ def test_error_if_plain_with_comment(self): def test_error_if_pop(self): input = """ inst(OP, (left, right -- res)) { - res = spam(left, right); + SPAM(left, right); + INPUTS_DEAD(); ERROR_IF(cond, label); + res = 0; } """ output = """ @@ -333,7 +404,36 @@ def test_error_if_pop(self): _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - res = spam(left, right); + SPAM(left, right); + if (cond) goto pop_2_label; + res = 0; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop_with_result(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond, label); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); if (cond) goto pop_2_label; stack_pointer[-2] = res; stack_pointer += -1; @@ -350,7 +450,7 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); @@ -388,10 +488,12 @@ def test_macro_instruction(self): } op(OP2, (extra/2, arg2, left, right -- res)) { res = op2(arg2, left, right); + INPUTS_DEAD(); } macro(OP) = OP1 + cache/2 + OP2; inst(OP3, (unused/5, arg2, left, right -- res)) { res = op3(arg2, left, right); + INPUTS_DEAD(); } family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; """ @@ -401,27 +503,31 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP); PREDICTED(OP); - _Py_CODEUNIT *this_instr = next_instr - 6; + _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef left; _PyStackRef right; _PyStackRef arg2; _PyStackRef res; // _OP1 - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 2 cache entries */ // OP2 - arg2 = stack_pointer[-3]; { + arg2 = stack_pointer[-3]; uint32_t extra = read_u32(&this_instr[4].cache); (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer[-3] = res; stack_pointer += -2; @@ -430,7 +536,7 @@ def test_macro_instruction(self): } TARGET(OP1) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); @@ -440,7 +546,9 @@ def test_macro_instruction(self): left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -457,7 +565,9 @@ def test_macro_instruction(self): right = stack_pointer[-1]; left = stack_pointer[-2]; arg2 = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); res = op3(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -469,7 +579,7 @@ def test_macro_instruction(self): def test_unused_caches(self): input = """ inst(OP, (unused/1, unused/2 --)) { - body(); + body; } """ output = """ @@ -479,7 +589,7 @@ def test_unused_caches(self): INSTRUCTION_STATS(OP); /* Skip 1 cache entry */ /* Skip 2 cache entries */ - body(); + body; DISPATCH(); } """ @@ -523,10 +633,40 @@ def test_pseudo_instruction_with_flags(self): """ self.run_cases_test(input, output) + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input(self): input = """ inst(OP, (below, values[oparg*2], above --)) { - spam(values, oparg); + SPAM(values, oparg); } """ output = """ @@ -536,7 +676,7 @@ def test_array_input(self): INSTRUCTION_STATS(OP); _PyStackRef *values; values = &stack_pointer[-1 - oparg*2]; - spam(values, oparg); + SPAM(values, oparg); stack_pointer += -2 - oparg*2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -547,7 +687,9 @@ def test_array_input(self): def test_array_output(self): input = """ inst(OP, (unused, unused -- below, values[oparg*3], above)) { - spam(values, oparg); + SPAM(values, oparg); + below = 0; + above = 0; } """ output = """ @@ -555,9 +697,13 @@ def test_array_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); + _PyStackRef below; _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-1]; - spam(values, oparg); + SPAM(values, oparg); + below = 0; + above = 0; stack_pointer[-2] = below; stack_pointer[-1 + oparg*3] = above; stack_pointer += oparg*3; @@ -570,7 +716,8 @@ def test_array_output(self): def test_array_input_output(self): input = """ inst(OP, (values[oparg] -- values[oparg], above)) { - spam(values, oparg); + SPAM(values, oparg); + above = 0; } """ output = """ @@ -579,8 +726,10 @@ def test_array_input_output(self): next_instr += 1; INSTRUCTION_STATS(OP); _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-oparg]; - spam(values, oparg); + SPAM(values, oparg); + above = 0; stack_pointer[0] = above; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -615,7 +764,10 @@ def test_array_error_if(self): def test_cond_effect(self): input = """ inst(OP, (aa, input if ((oparg & 1) == 1), cc -- xx, output if (oparg & 2), zz)) { - output = spam(oparg, aa, cc, input); + output = SPAM(oparg, aa, cc, input); + INPUTS_DEAD(); + xx = 0; + zz = 0; } """ output = """ @@ -626,11 +778,15 @@ def test_cond_effect(self): _PyStackRef aa; _PyStackRef input = PyStackRef_NULL; _PyStackRef cc; + _PyStackRef xx; _PyStackRef output = PyStackRef_NULL; + _PyStackRef zz; cc = stack_pointer[-1]; if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; - output = spam(oparg, aa, cc, input); + output = SPAM(oparg, aa, cc, input); + xx = 0; + zz = 0; stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)] = xx; if (oparg & 2) stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)] = output; stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0)] = zz; @@ -644,11 +800,14 @@ def test_cond_effect(self): def test_macro_cond_effect(self): input = """ op(A, (left, middle, right --)) { - use(left, middle, right); + USE(left, middle, right); + INPUTS_DEAD(); } op(B, (-- deep, extra if (oparg), res)) { + deep = -1; res = 0; extra = 1; + INPUTS_DEAD(); } macro(M) = A + B; """ @@ -660,17 +819,19 @@ def test_macro_cond_effect(self): _PyStackRef left; _PyStackRef middle; _PyStackRef right; + _PyStackRef deep; _PyStackRef extra = PyStackRef_NULL; _PyStackRef res; // A - right = stack_pointer[-1]; - middle = stack_pointer[-2]; - left = stack_pointer[-3]; { - use(left, middle, right); + right = stack_pointer[-1]; + middle = stack_pointer[-2]; + left = stack_pointer[-3]; + USE(left, middle, right); } // B { + deep = -1; res = 0; extra = 1; } @@ -687,10 +848,10 @@ def test_macro_cond_effect(self): def test_macro_push_push(self): input = """ op(A, (-- val1)) { - val1 = spam(); + val1 = SPAM(); } op(B, (-- val2)) { - val2 = spam(); + val2 = SPAM(); } macro(M) = A + B; """ @@ -703,11 +864,11 @@ def test_macro_push_push(self): _PyStackRef val2; // A { - val1 = spam(); + val1 = SPAM(); } // B { - val2 = spam(); + val2 = SPAM(); } stack_pointer[0] = val1; stack_pointer[1] = val2; @@ -721,10 +882,10 @@ def test_macro_push_push(self): def test_override_inst(self): input = """ inst(OP, (--)) { - spam(); + spam; } override inst(OP, (--)) { - ham(); + ham; } """ output = """ @@ -732,7 +893,7 @@ def test_override_inst(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - ham(); + ham; DISPATCH(); } """ @@ -741,11 +902,11 @@ def test_override_inst(self): def test_override_op(self): input = """ op(OP, (--)) { - spam(); + spam; } macro(M) = OP; override op(OP, (--)) { - ham(); + ham; } """ output = """ @@ -753,7 +914,7 @@ def test_override_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - ham(); + ham; DISPATCH(); } """ @@ -762,7 +923,7 @@ def test_override_op(self): def test_annotated_inst(self): input = """ pure inst(OP, (--)) { - ham(); + ham; } """ output = """ @@ -770,7 +931,7 @@ def test_annotated_inst(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - ham(); + ham; DISPATCH(); } """ @@ -779,7 +940,7 @@ def test_annotated_inst(self): def test_annotated_op(self): input = """ pure op(OP, (--)) { - spam(); + SPAM(); } macro(M) = OP; """ @@ -788,7 +949,7 @@ def test_annotated_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - spam(); + SPAM(); DISPATCH(); } """ @@ -796,7 +957,7 @@ def test_annotated_op(self): input = """ pure register specializing op(OP, (--)) { - spam(); + SPAM(); } macro(M) = OP; """ @@ -810,7 +971,7 @@ def test_deopt_and_exit(self): } """ output = "" - with self.assertRaises(Exception): + with self.assertRaises(SyntaxError): self.run_cases_test(input, output) def test_array_of_one(self): @@ -838,6 +999,7 @@ def test_pointer_to_stackref(self): input = """ inst(OP, (arg: _PyStackRef * -- out)) { out = *arg; + DEAD(arg); } """ output = """ @@ -892,14 +1054,14 @@ def test_unused_named_values(self): def test_used_unused_used(self): input = """ op(FIRST, (w -- w)) { - use(w); + USE(w); } op(SECOND, (x -- x)) { } op(THIRD, (y -- y)) { - use(y); + USE(y); } macro(TEST) = FIRST + SECOND + THIRD; @@ -912,17 +1074,17 @@ def test_used_unused_used(self): _PyStackRef w; _PyStackRef y; // FIRST - w = stack_pointer[-1]; { - use(w); + w = stack_pointer[-1]; + USE(w); } // SECOND { } // THIRD - y = w; { - use(y); + y = w; + USE(y); } DISPATCH(); } @@ -935,11 +1097,11 @@ def test_unused_used_used(self): } op(SECOND, (x -- x)) { - use(x); + USE(x); } op(THIRD, (y -- y)) { - use(y); + USE(y); } macro(TEST) = FIRST + SECOND + THIRD; @@ -955,14 +1117,14 @@ def test_unused_used_used(self): { } // SECOND - x = stack_pointer[-1]; { - use(x); + x = stack_pointer[-1]; + USE(x); } // THIRD - y = x; { - use(y); + y = x; + USE(y); } DISPATCH(); } @@ -977,7 +1139,8 @@ def test_flush(self): } op(SECOND, (a, b -- )) { - use(a, b); + USE(a, b); + INPUTS_DEAD(); } macro(TEST) = FIRST + flush + SECOND; @@ -1000,10 +1163,8 @@ def test_flush(self): stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); // SECOND - b = stack_pointer[-1]; - a = stack_pointer[-2]; { - use(a, b); + USE(a, b); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1017,14 +1178,16 @@ def test_pop_on_error_peeks(self): input = """ op(FIRST, (x, y -- a, b)) { a = x; + DEAD(x); b = y; + DEAD(y); } op(SECOND, (a, b -- a, b)) { } op(THIRD, (j, k --)) { - j,k; // Mark j and k as used + INPUTS_DEAD(); // Mark j and k as used ERROR_IF(cond, error); } @@ -1039,12 +1202,10 @@ def test_pop_on_error_peeks(self): _PyStackRef y; _PyStackRef a; _PyStackRef b; - _PyStackRef j; - _PyStackRef k; // FIRST - y = stack_pointer[-1]; - x = stack_pointer[-2]; { + y = stack_pointer[-1]; + x = stack_pointer[-2]; a = x; b = y; } @@ -1052,10 +1213,8 @@ def test_pop_on_error_peeks(self): { } // THIRD - k = b; - j = a; { - j,k; // Mark j and k as used + // Mark j and k as used if (cond) goto pop_2_error; } stack_pointer += -2; @@ -1096,7 +1255,8 @@ def test_push_then_error(self): b = 1; if (cond) { stack_pointer[0] = a; - stack_pointer += 1; + stack_pointer[1] = b; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); goto error; } @@ -1148,6 +1308,69 @@ def test_array_size_inconsistency(self): with self.assertRaises(SyntaxError): self.run_cases_test(input, output) + def test_stack_save_reload(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_reload_only(self): + + input = """ + inst(BALANCED, ( -- )) { + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_only(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: @@ -1202,7 +1425,7 @@ def run_cases_test(self, input: str, input2: str, expected: str): def test_overridden_abstract(self): input = """ pure op(OP, (--)) { - spam(); + SPAM(); } """ input2 = """ @@ -1221,22 +1444,23 @@ def test_overridden_abstract(self): def test_overridden_abstract_args(self): input = """ pure op(OP, (arg1 -- out)) { - spam(); + out = SPAM(arg1); } op(OP2, (arg1 -- out)) { - eggs(); + out = EGGS(arg1); } """ input2 = """ op(OP, (arg1 -- out)) { - eggs(); + out = EGGS(arg1); } """ output = """ case OP: { _Py_UopsSymbol *arg1; _Py_UopsSymbol *out; - eggs(); + arg1 = stack_pointer[-1]; + out = EGGS(arg1); stack_pointer[-1] = out; break; } @@ -1253,7 +1477,7 @@ def test_overridden_abstract_args(self): def test_no_overridden_case(self): input = """ pure op(OP, (arg1 -- out)) { - spam(); + out = SPAM(arg1); } pure op(OP2, (arg1 -- out)) { @@ -1262,6 +1486,7 @@ def test_no_overridden_case(self): """ input2 = """ pure op(OP2, (arg1 -- out)) { + out = NULL; } """ output = """ @@ -1273,8 +1498,8 @@ def test_no_overridden_case(self): } case OP2: { - _Py_UopsSymbol *arg1; _Py_UopsSymbol *out; + out = NULL; stack_pointer[-1] = out; break; } @@ -1284,7 +1509,7 @@ def test_no_overridden_case(self): def test_missing_override_failure(self): input = """ pure op(OP, (arg1 -- out)) { - spam(); + SPAM(); } """ input2 = """ diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index bf04b3fecf7057..6d2593cb4cf228 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -156,6 +156,10 @@ def test_exists(self): self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) + # Keyword arguments are accepted + self.assertIs(self.pathmodule.exists(path=filename), True) + self.assertIs(self.pathmodule.lexists(path=filename), True) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") def test_exists_fd(self): diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index ae384c3849d49e..bf6e1703db8451 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -713,6 +713,17 @@ def test_compress_mtime(self): f.read(1) # to set mtime attribute self.assertEqual(f.mtime, mtime) + def test_compress_mtime_default(self): + # test for gh-125260 + datac = gzip.compress(data1, mtime=0) + datac2 = gzip.compress(data1) + self.assertEqual(datac, datac2) + datac3 = gzip.compress(data1, mtime=None) + self.assertNotEqual(datac, datac3) + with gzip.GzipFile(fileobj=io.BytesIO(datac3), mode="rb") as f: + f.read(1) # to set mtime attribute + self.assertGreater(f.mtime, 1) + def test_compress_correct_level(self): for mtime in (0, 42): with self.subTest(mtime=mtime): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 1fd75d0a3f4c7b..a6509fc3ba0eae 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -57,7 +57,7 @@ def timevalues(self): timezone(timedelta(0, 2 * 60 * 60))), '"18-May-2033 05:33:20 +0200"'] - @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') # DST rules included to work around quirk where the Gnu C library may not # otherwise restore the previous time zone @run_with_tz('STD-1DST,M3.2.0,M11.1.0') diff --git a/Lib/test/test_inspect/inspect_deferred_annotations.py b/Lib/test/test_inspect/inspect_deferred_annotations.py new file mode 100644 index 00000000000000..bb59ef1035b3c1 --- /dev/null +++ b/Lib/test/test_inspect/inspect_deferred_annotations.py @@ -0,0 +1,2 @@ +def f(x: undefined): + pass diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43e9f852022934..43fda6622537fc 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -357,3 +357,15 @@ class td354(typing.TypedDict): # line 358 td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 81188ad4d1fbe1..9fa6d23d15f06a 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1,3 +1,4 @@ +from annotationlib import Format, ForwardRef import asyncio import builtins import collections @@ -22,7 +23,6 @@ import types import tempfile import textwrap -from typing import Unpack import unicodedata import unittest import unittest.mock @@ -46,6 +46,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_deferred_annotations # Functions tested in this suite: @@ -835,6 +836,47 @@ class C: nonlocal __firstlineno__ self.assertRaises(OSError, inspect.getsource, C) +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -947,6 +989,11 @@ def test_typeddict(self): self.assertSourceEqual(mod2.td354, 354, 356) self.assertRaises(OSError, inspect.getsource, mod2.td359) + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1010,7 +1057,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -1018,9 +1065,20 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) @@ -3341,7 +3399,7 @@ def foo(cls, *, arg): ...)) def test_signature_on_partial(self): - from functools import partial + from functools import partial, Placeholder def test(): pass @@ -3396,6 +3454,25 @@ def test(a, b, *, c, d): ('d', ..., ..., "keyword_only")), ...)) + # With Placeholder + self.assertEqual(self.signature(partial(test, Placeholder, 1)), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), + ((('a', ..., ..., "positional_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder + self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)), + ((('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + def test(a, *args, b, **kwargs): pass @@ -3443,6 +3520,15 @@ def test(a, *args, b, **kwargs): ('kwargs', ..., ..., "var_keyword")), ...)) + # With Placeholder + p = partial(test, Placeholder, Placeholder, 1, b=0, test=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + def test(a, b, c:int) -> 42: pass @@ -3547,6 +3633,34 @@ def foo(a, b, /, c, d, **kwargs): ('kwargs', ..., ..., 'var_keyword')), ...)) + # Positional only With Placeholder + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Optionals Positional With Placeholder + def foo(a=0, b=1, /, c=2, d=3): + pass + + # Positional + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only")), + ...)) + + # Positional or Keyword - transformed to positional + p = partial(foo, Placeholder, 1, Placeholder, 1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "positional_only")), + ...)) + def test_signature_on_partialmethod(self): from functools import partialmethod @@ -3559,18 +3673,32 @@ def test(): inspect.signature(Spam.ham) class Spam: - def test(it, a, *, c) -> 'spam': + def test(it, a, b, *, c) -> 'spam': pass ham = partialmethod(test, c=1) + bar = partialmethod(test, functools.Placeholder, 1, c=1) self.assertEqual(self.signature(Spam.ham, eval_str=False), ((('it', ..., ..., 'positional_or_keyword'), ('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), ('c', 1, ..., 'keyword_only')), 'spam')) self.assertEqual(self.signature(Spam().ham, eval_str=False), ((('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + # With Placeholder + self.assertEqual(self.signature(Spam.bar, eval_str=False), + ((('it', ..., ..., 'positional_only'), + ('a', ..., ..., 'positional_only'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + self.assertEqual(self.signature(Spam().bar, eval_str=False), + ((('a', ..., ..., 'positional_only'), ('c', 1, ..., 'keyword_only')), 'spam')) @@ -4495,6 +4623,18 @@ def func( expected_multiline, ) + def test_signature_format_unquote(self): + def func(x: 'int') -> 'str': ... + + self.assertEqual( + inspect.signature(func).format(), + "(x: 'int') -> 'str'" + ) + self.assertEqual( + inspect.signature(func).format(quote_annotation_strings=False), + "(x: int) -> str" + ) + def test_signature_replace_parameters(self): def test(a, b) -> 42: pass @@ -4727,6 +4867,26 @@ def test_signature_eval_str(self): par('b', PORK, annotation=tuple), ))) + def test_signature_annotation_format(self): + ida = inspect_deferred_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func=signature_func): + self.assertEqual( + signature_func(ida.f, annotation_format=Format.STRING), + sig([par("x", PORK, annotation="undefined")]) + ) + self.assertEqual( + signature_func(ida.f, annotation_format=Format.FORWARDREF), + sig([par("x", PORK, annotation=ForwardRef("undefined"))]) + ) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f, annotation_format=Format.VALUE) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct @@ -4752,38 +4912,6 @@ def foo(): pass self.assertEqual(signature_func(foo), inspect.Signature()) self.assertEqual(inspect.get_annotations(foo), {}) - def test_signature_as_str(self): - self.maxDiff = None - class S: - __signature__ = '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - def test_signature_as_callable(self): - # __signature__ should be either a staticmethod or a bound classmethod - class S: - @classmethod - def __signature__(cls): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - class S: - @staticmethod - def __signature__(): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - def test_signature_on_derived_classes(self): # gh-105080: Make sure that signatures are consistent on derived classes diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 6820dce3f12620..8469de998ba014 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1249,10 +1249,11 @@ def test_tee(self): self.assertEqual(len(result), n) self.assertEqual([list(x) for x in result], [list('abc')]*n) - # tee pass-through to copyable iterator + # tee objects are independent (see bug gh-123884) a, b = tee('abc') c, d = tee(a) - self.assertTrue(a is c) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) # test tee_new t1, t2 = tee('abc') @@ -1759,21 +1760,36 @@ def test_tee_recipe(self): def tee(iterable, n=2): if n < 0: - raise ValueError('n must be >= 0') - iterator = iter(iterable) - shared_link = [None, None] - return tuple(_tee(iterator, shared_link) for _ in range(n)) + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] - def _tee(iterator, link): - try: - while True: - if link[1] is None: - link[0] = next(iterator) - link[1] = [None, None] - value, link = link - yield value - except StopIteration: - return + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value # End tee() recipe ############################################# @@ -1819,12 +1835,10 @@ def _tee(iterator, link): self.assertRaises(TypeError, tee, [1,2], 'x') self.assertRaises(TypeError, tee, [1,2], 3, 'x') - # Tests not applicable to the tee() recipe - if False: - # tee object should be instantiable - a, b = tee('abc') - c = type(a)('def') - self.assertEqual(list(c), list('def')) + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) # test long-lagged and multi-way split a, b, c = tee(range(2000), 3) @@ -1845,21 +1859,19 @@ def _tee(iterator, link): self.assertEqual(len(result), n) self.assertEqual([list(x) for x in result], [list('abc')]*n) + # tee objects are independent (see bug gh-123884) + a, b = tee('abc') + c, d = tee(a) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) - # Tests not applicable to the tee() recipe - if False: - # tee pass-through to copyable iterator - a, b = tee('abc') - c, d = tee(a) - self.assertTrue(a is c) - - # test tee_new - t1, t2 = tee('abc') - tnew = type(t1) - self.assertRaises(TypeError, tnew) - self.assertRaises(TypeError, tnew, 10) - t3 = tnew(t1) - self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) # test that tee objects are weak referencable a, b = tee(range(10)) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 230ba954cd286d..d4ceb7c8dc0b41 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -2377,16 +2377,22 @@ def __getattr__(self, attribute): return getattr(queue, attribute) class CustomQueueFakeProtocol(CustomQueueProtocol): - # An object implementing the Queue API (incorrect signatures). + # An object implementing the minimial Queue API for + # the logging module but with incorrect signatures. + # # The object will be considered a valid queue class since we # do not check the signatures (only callability of methods) # but will NOT be usable in production since a TypeError will - # be raised due to a missing argument. - def empty(self, x): + # be raised due to the extra argument in 'put_nowait'. + def put_nowait(self): pass class CustomQueueWrongProtocol(CustomQueueProtocol): - empty = None + put_nowait = None + +class MinimalQueueProtocol: + def put_nowait(self, x): pass + def get(self): pass def queueMaker(): return queue.Queue() @@ -3946,56 +3952,70 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + def _apply_simple_queue_listener_configuration(self, qspec): + self.apply_config({ + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, + }, + }) + @threading_helper.requires_working_threading() @support.requires_subprocess() @patch("multiprocessing.Manager") def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): - # gh-120868, gh-121723 - - from multiprocessing import Queue as MQ - - q1 = {"()": "queue.Queue", "maxsize": -1} - q2 = MQ() - q3 = queue.Queue() - # CustomQueueFakeProtocol passes the checks but will not be usable - # since the signatures are incompatible. Checking the Queue API - # without testing the type of the actual queue is a trade-off - # between usability and the work we need to do in order to safely - # check that the queue object correctly implements the API. - q4 = CustomQueueFakeProtocol() - - for qspec in (q1, q2, q3, q4): - self.apply_config( - { - "version": 1, - "handlers": { - "queue_listener": { - "class": "logging.handlers.QueueHandler", - "queue": qspec, - }, - }, - } - ) - manager.assert_not_called() + # gh-120868, gh-121723, gh-124653 + + for qspec in [ + {"()": "queue.Queue", "maxsize": -1}, + queue.Queue(), + # queue.SimpleQueue does not inherit from queue.Queue + queue.SimpleQueue(), + # CustomQueueFakeProtocol passes the checks but will not be usable + # since the signatures are incompatible. Checking the Queue API + # without testing the type of the actual queue is a trade-off + # between usability and the work we need to do in order to safely + # check that the queue object correctly implements the API. + CustomQueueFakeProtocol(), + MinimalQueueProtocol(), + ]: + with self.subTest(qspec=qspec): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() @patch("multiprocessing.Manager") def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): # gh-120868, gh-121723 for qspec in [object(), CustomQueueWrongProtocol()]: - with self.assertRaises(ValueError): - self.apply_config( - { - "version": 1, - "handlers": { - "queue_listener": { - "class": "logging.handlers.QueueHandler", - "queue": qspec, - }, - }, - } - ) - manager.assert_not_called() + with self.subTest(qspec=qspec), self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_reject_simple_queue_handler_multiprocessing_context(self): + # multiprocessing.SimpleQueue does not implement 'put_nowait' + # and thus cannot be used as a queue-like object (gh-124653) + + import multiprocessing + + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + qspec = ctx.SimpleQueue() + with self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) @skip_if_tsan_fork @support.requires_subprocess() diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index a3eebc97ada23b..e2e2a419c7778c 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2722,7 +2722,7 @@ def test_fma_infinities(self): # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 # properly: it doesn't use the right sign when the result is zero. @unittest.skipIf( - sys.platform.startswith(("freebsd", "wasi")) + sys.platform.startswith(("freebsd", "wasi", "netbsd")) or (sys.platform == "android" and platform.machine() == "x86_64"), f"this platform doesn't implement IEE 754-2008 properly") def test_fma_zero_result(self): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 351f1067c10343..1b06816214e7d6 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -46,10 +46,14 @@ def nth_line(func, offset): class MonitoringBasicTest(unittest.TestCase): + def tearDown(self): + sys.monitoring.free_tool_id(TEST_TOOL) + def test_has_objects(self): m = sys.monitoring m.events m.use_tool_id + m.clear_tool_id m.free_tool_id m.get_tool m.get_events @@ -77,6 +81,43 @@ def test_tool(self): with self.assertRaises(ValueError): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + def test_clear(self): + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, lambda *args: events.append(args)) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + def f(): + a = 1 + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + + f() + sys.monitoring.clear_tool_id(TEST_TOOL) + f() + + # the first f() should trigger a PY_START and a LINE event + # the second f() after clear_tool_id should not trigger any event + # the callback function should be cleared as well + self.assertEqual(len(events), 2) + callback = sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertIs(callback, None) + + sys.monitoring.free_tool_id(TEST_TOOL) + + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + f() + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + f() + # the first f() should trigger a LINE event, and even if we use the + # tool id immediately after freeing it, the second f() should not + # trigger any event + self.assertEqual(len(events), 1) + sys.monitoring.free_tool_id(TEST_TOOL) + class MonitoringTestBase: @@ -744,8 +785,6 @@ def get_events(self, func, tool, recorders): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): events = self.get_events(func, tool, recorders) - if events != expected: - print(events, file = sys.stderr) self.assertEqual(events, expected) def check_balanced(self, func, recorders): diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index b47b4a194cfaa9..c7104bfda90f6c 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -131,6 +131,15 @@ def test_constructor_nested(self): self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) self.assertEqual(P(P('./a:b')), P('./a:b')) + @needs_windows + def test_constructor_nested_foreign_flavour(self): + # See GH-125069. + p1 = pathlib.PurePosixPath('b/c:\\d') + p2 = pathlib.PurePosixPath('b/', 'c:\\d') + self.assertEqual(p1, p2) + self.assertEqual(self.cls(p1), self.cls('b/c:/d')) + self.assertEqual(self.cls(p2), self.cls('b/c:/d')) + def _check_parse_path(self, raw_path, *expected): sep = self.parser.sep actual = self.cls._parse_path(raw_path.replace('/', sep)) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index db7d1b1e9cd935..3dc65fdfc03409 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -363,6 +363,54 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_commands(): + """Test the commands command of pdb. + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... print(1) + ... print(2) + ... print(3) + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'commands', + ... 'silent', # suppress the frame status output + ... 'p "hello"', + ... 'end', + ... 'b 4', + ... 'commands', + ... 'until 5', # no output, should stop at line 5 + ... 'continue', # hit breakpoint at line 3 + ... '', # repeat continue, hit breakpoint at line 4 then `until` to line 5 + ... '', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) commands + (com) silent + (com) p "hello" + (com) end + (Pdb) b 4 + Breakpoint 2 at :4 + (Pdb) commands + (com) until 5 + (Pdb) continue + 'hello' + (Pdb) + 1 + 2 + > (5)test_function() + -> print(3) + (Pdb) + 3 + """ + def test_pdb_breakpoint_with_filename(): """Breakpoints with filename:lineno @@ -470,6 +518,43 @@ def test_pdb_breakpoints_preserved_across_interactive_sessions(): (Pdb) continue """ +def test_pdb_break_anywhere(): + """Test break_anywhere() method of Pdb. + + >>> def outer(): + ... def inner(): + ... import pdb + ... import sys + ... p = pdb.Pdb(nosigint=True, readrc=False) + ... p.set_trace() + ... frame = sys._getframe() + ... print(p.break_anywhere(frame)) # inner + ... print(p.break_anywhere(frame.f_back)) # outer + ... print(p.break_anywhere(frame.f_back.f_back)) # caller + ... inner() + + >>> def caller(): + ... outer() + + >>> def test_function(): + ... caller() + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'c', + ... ]): + ... test_function() + > (6)inner() + -> p.set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) c + True + False + False + """ + def test_pdb_pp_repr_exc(): """Test that do_p/do_pp do not swallow exceptions. @@ -901,6 +986,38 @@ def test_pdb_where_command(): (Pdb) continue """ +def test_pdb_restart_command(): + """Test restart command + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + ... x = 1 + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'restart', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + (Pdb) restart + *** run/restart command is disabled when pdb is running in inline mode. + Use the command line interface to enable restarting your program + e.g. "python -m pdb myscript.py" + (Pdb) continue + """ + +def test_pdb_commands_with_set_trace(): + """Test that commands can be passed to Pdb.set_trace() + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace(commands=['p x', 'c']) + + >>> test_function() + 1 + """ + # skip this test if sys.flags.no_site = True; # exit() isn't defined unless there's a site module. @@ -3276,6 +3393,20 @@ def test_issue26053(self): self.assertRegex(res, "Restarting .* with arguments:\na b c") self.assertRegex(res, "Restarting .* with arguments:\nd e f") + def test_step_into_botframe(self): + # gh-125422 + # pdb should not be able to step into the botframe (bdb.py) + script = "x = 1" + commands = """ + step + step + step + quit + """ + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn("The program finished", stdout) + self.assertNotIn("bdb.py", stdout) + def test_pdbrc_basic(self): script = textwrap.dedent(""" a = 1 diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index b68a55259c62e1..b55d441759eb69 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -62,11 +62,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -130,11 +132,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -179,11 +183,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -229,7 +235,7 @@ def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False - return "no-omit-frame-pointer" in cflags and "_Py_JIT" not in cflags + return "no-omit-frame-pointer" in cflags def perf_command_works(): @@ -260,8 +266,9 @@ def perf_command_works(): "-c", 'print("hello")', ) + env = {**os.environ, "PYTHON_JIT": "0"} stdout = subprocess.check_output( - cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env ) except (subprocess.SubprocessError, OSError): return False @@ -273,11 +280,10 @@ def perf_command_works(): def run_perf(cwd, *args, use_jit=False, **env_vars): + env = os.environ.copy() if env_vars: - env = os.environ.copy() env.update(env_vars) - else: - env = None + env["PYTHON_JIT"] = "0" output_file = cwd + "/perf_output.perf" if not use_jit: base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") @@ -382,6 +388,7 @@ def baz(n): self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) + @unittest.skipUnless(perf_command_works(), "perf command doesn't work") @unittest.skipUnless( is_unwinding_reliable_with_frame_pointers(), @@ -445,11 +452,13 @@ def compile_trampolines_for_all_functions(): with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -494,7 +503,9 @@ def _is_perf_version_at_least(major, minor): @unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(_is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug") +@unittest.skipUnless( + _is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug" +) class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): def run_perf(self, script_dir, script, activate_trampoline=True): if activate_trampoline: diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2dba077cdea6a7..2a4d3ab73db608 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -463,6 +463,14 @@ class BinaryInteger(enum.IntEnum): doc = pydoc.render_doc(BinaryInteger) self.assertIn('BinaryInteger.zero', doc) + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + def test_mixed_case_module_names_are_lower_cased(self): # issue16484 doc_link = get_pydoc_link(xml.etree.ElementTree) @@ -1065,7 +1073,7 @@ def __init__(self, class A(builtins.object) | A( - | arg1: collections.abc.Callable[[int, int, int], str], + | arg1: Callable[[int, int, int], str], | arg2: Literal['some value', 'other value'], | arg3: Annotated[int, 'some docs about this type'] | ) -> None @@ -1074,7 +1082,7 @@ class A(builtins.object) | | __init__( | self, - | arg1: collections.abc.Callable[[int, int, int], str], + | arg1: Callable[[int, int, int], str], | arg2: Literal['some value', 'other value'], | arg3: Annotated[int, 'some docs about this type'] | ) -> None @@ -1101,7 +1109,7 @@ def func( self.assertEqual(doc, '''Python Library Documentation: function func in module %s func( - arg1: collections.abc.Callable[[typing.Annotated[int, 'Some doc']], str], + arg1: Callable[[Annotated[int, 'Some doc']], str], arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] ) -> Annotated[int, 'Some other'] ''' % __name__) @@ -1386,8 +1394,8 @@ def foo(data: typing.List[typing.Any], T = typing.TypeVar('T') class C(typing.Generic[T], typing.Mapping[int, str]): ... self.assertEqual(pydoc.render_doc(foo).splitlines()[-1], - 'f\x08fo\x08oo\x08o(data: List[Any], x: int)' - ' -> Iterator[Tuple[int, Any]]') + 'f\x08fo\x08oo\x08o(data: typing.List[typing.Any], x: int)' + ' -> typing.Iterator[typing.Tuple[int, typing.Any]]') self.assertEqual(pydoc.render_doc(C).splitlines()[2], 'class C\x08C(collections.abc.Mapping, typing.Generic)') diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index b7adaffbac0e22..0c6df4e5dae869 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -119,13 +119,38 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): def test_no_active_future(self): console = InteractiveColoredConsole() - source = "x: int = 1; print(__annotate__(1))" + source = dedent("""\ + x: int = 1 + print(__annotate__(1)) + """) f = io.StringIO() with contextlib.redirect_stdout(f): result = console.runsource(source) self.assertFalse(result) self.assertEqual(f.getvalue(), "{'x': }\n") + def test_future_annotations(self): + console = InteractiveColoredConsole() + source = dedent("""\ + from __future__ import annotations + def g(x: int): ... + print(g.__annotations__) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': 'int'}\n") + + def test_future_barry_as_flufl(self): + console = InteractiveColoredConsole() + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource("from __future__ import barry_as_FLUFL\n") + result = console.runsource("""print("black" <> 'blue')\n""") + self.assertFalse(result) + self.assertEqual(f.getvalue(), "True\n") + class TestMoreLines(unittest.TestCase): def test_invalid_syntax_single_line(self): diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index e816de3720670f..1a76832386bf1d 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -8,7 +8,7 @@ import subprocess import sys import tempfile -from unittest import TestCase, skipUnless +from unittest import TestCase, skipUnless, skipIf from unittest.mock import patch from test.support import force_not_colorized from test.support import SHORT_TIMEOUT @@ -35,6 +35,94 @@ except ImportError: pty = None + +class ReplTestCase(TestCase): + def run_repl( + self, + repl_input: str | list[str], + env: dict | None = None, + *, + cmdline_args: list[str] | None = None, + cwd: str | None = None, + ) -> tuple[str, int]: + temp_dir = None + if cwd is None: + temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + cwd = temp_dir.name + try: + return self._run_repl( + repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd + ) + finally: + if temp_dir is not None: + temp_dir.cleanup() + + def _run_repl( + self, + repl_input: str | list[str], + *, + env: dict | None, + cmdline_args: list[str] | None, + cwd: str, + ) -> tuple[str, int]: + assert pty + master_fd, slave_fd = pty.openpty() + cmd = [sys.executable, "-i", "-u"] + if env is None: + cmd.append("-I") + elif "PYTHON_HISTORY" not in env: + env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") + if cmdline_args is not None: + cmd.extend(cmdline_args) + + try: + import termios + except ModuleNotFoundError: + pass + else: + term_attr = termios.tcgetattr(slave_fd) + term_attr[6][termios.VREPRINT] = 0 # pass through CTRL-R + term_attr[6][termios.VINTR] = 0 # pass through CTRL-C + termios.tcsetattr(slave_fd, termios.TCSANOW, term_attr) + + process = subprocess.Popen( + cmd, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + cwd=cwd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + os.close(slave_fd) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + else: + os.close(master_fd) + process.kill() + self.fail(f"Timeout while waiting for output, got: {''.join(output)}") + + os.close(master_fd) + try: + exit_code = process.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.wait() + return "".join(output), exit_code + + class TestCursorPosition(TestCase): def prepare_reader(self, events): console = FakeConsole(events) @@ -968,7 +1056,20 @@ def test_bracketed_paste_single_line(self): @skipUnless(pty, "requires pty") -class TestMain(TestCase): +class TestDumbTerminal(ReplTestCase): + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can't use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + +@skipUnless(pty, "requires pty") +@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +class TestMain(ReplTestCase): def setUp(self): # Cleanup from PYTHON* variables to isolate from local # user settings, see #121359. Such variables should be @@ -979,7 +1080,7 @@ def setUp(self): @force_not_colorized def test_exposed_globals_in_repl(self): - pre = "['__annotations__', '__builtins__'" + pre = "['__builtins__'" post = "'__loader__', '__name__', '__package__', '__spec__']" output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) if "can't use pyrepl" in output: @@ -1078,15 +1179,6 @@ def test_inspect_keeps_globals_from_inspected_module(self): } self._run_repl_globals_test(expectations, as_module=True) - def test_dumb_terminal_exits_cleanly(self): - env = os.environ.copy() - env.update({"TERM": "dumb"}) - output, exit_code = self.run_repl("exit()\n", env=env) - self.assertEqual(exit_code, 0) - self.assertIn("warning: can't use pyrepl", output) - self.assertNotIn("Exception", output) - self.assertNotIn("Traceback", output) - @force_not_colorized def test_python_basic_repl(self): env = os.environ.copy() @@ -1112,6 +1204,18 @@ def test_python_basic_repl(self): self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) + # The site module must not load _pyrepl if PYTHON_BASIC_REPL is set + commands = ("import sys\n" + "print('_pyrepl' in sys.modules)\n" + "exit()\n") + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + @force_not_colorized def test_bad_sys_excepthook_doesnt_crash_pyrepl(self): env = os.environ.copy() @@ -1209,80 +1313,6 @@ def test_proper_tracebacklimit(self): self.assertIn("in x3", output) self.assertIn("in ", output) - def run_repl( - self, - repl_input: str | list[str], - env: dict | None = None, - *, - cmdline_args: list[str] | None = None, - cwd: str | None = None, - ) -> tuple[str, int]: - temp_dir = None - if cwd is None: - temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) - cwd = temp_dir.name - try: - return self._run_repl( - repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd - ) - finally: - if temp_dir is not None: - temp_dir.cleanup() - - def _run_repl( - self, - repl_input: str | list[str], - *, - env: dict | None, - cmdline_args: list[str] | None, - cwd: str, - ) -> tuple[str, int]: - assert pty - master_fd, slave_fd = pty.openpty() - cmd = [sys.executable, "-i", "-u"] - if env is None: - cmd.append("-I") - elif "PYTHON_HISTORY" not in env: - env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") - if cmdline_args is not None: - cmd.extend(cmdline_args) - process = subprocess.Popen( - cmd, - stdin=slave_fd, - stdout=slave_fd, - stderr=slave_fd, - cwd=cwd, - text=True, - close_fds=True, - env=env if env else os.environ, - ) - os.close(slave_fd) - if isinstance(repl_input, list): - repl_input = "\n".join(repl_input) + "\n" - os.write(master_fd, repl_input.encode("utf-8")) - - output = [] - while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: - try: - data = os.read(master_fd, 1024).decode("utf-8") - if not data: - break - except OSError: - break - output.append(data) - else: - os.close(master_fd) - process.kill() - self.fail(f"Timeout while waiting for output, got: {''.join(output)}") - - os.close(master_fd) - try: - exit_code = process.wait(timeout=SHORT_TIMEOUT) - except subprocess.TimeoutExpired: - process.kill() - exit_code = process.wait() - return "".join(output), exit_code - def test_readline_history_file(self): # skip, if readline module is not available readline = import_module('readline') @@ -1305,3 +1335,7 @@ def test_readline_history_file(self): output, exit_code = self.run_repl("exit\n", env=env) self.assertEqual(exit_code, 0) self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text()) + + def test_keyboard_interrupt_after_isearch(self): + output, exit_code = self.run_repl(["\x12", "\x03", "exit"]) + self.assertEqual(exit_code, 0) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 7a7285a1a2fcfd..e764e60560db23 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -291,5 +291,42 @@ def f(): self.assertEqual(traceback_lines, expected_lines) +class TestAsyncioREPLContextVars(unittest.TestCase): + def test_toplevel_contextvars_sync(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar("var", default="failed") + var.set("ok") + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = dedent(""" + print(f"toplevel contextvar test: {var.get()}") + """) + p.stdin.write(user_input2) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + def test_toplevel_contextvars_async(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar('var', default='failed') + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = "async def set_var(): var.set('ok')\n" + p.stdin.write(user_input2) + user_input3 = "await set_var()\n" + p.stdin.write(user_input3) + user_input4 = "print(f'toplevel contextvar test: {var.get()}')\n" + p.stdin.write(user_input4) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 80e1d73b6b2aab..37e54d23b22516 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1909,7 +1909,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 303f9e03b5383f..d014a9ce841607 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -34,7 +34,9 @@ def expect_failure(self, *args): def test_cli_help(self): out = self.expect_success("-h") - self.assertIn("usage: python -m sqlite3", out) + self.assertIn("usage: ", out) + self.assertIn(" [-h] [-v] [filename] [sql]", out) + self.assertIn("Python sqlite3 CLI", out) def test_cli_version(self): out = self.expect_success("-v") diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index d508f238f84fb5..550cea41976441 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -10,6 +10,7 @@ class DumpTests(MemoryDatabaseMixin, unittest.TestCase): def test_table_dump(self): expected_sqls = [ + "PRAGMA foreign_keys=OFF;", """CREATE TABLE "index"("index" blob);""" , """INSERT INTO "index" VALUES(X'01');""" @@ -48,7 +49,7 @@ def test_table_dump(self): expected_sqls = [ "PRAGMA foreign_keys=OFF;", "BEGIN TRANSACTION;", - *expected_sqls, + *expected_sqls[1:], "COMMIT;", ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9c415bd7d1c4e4..b93fa0ed99f8ce 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3,6 +3,7 @@ import sys import unittest import unittest.mock +from ast import literal_eval from test import support from test.support import import_helper from test.support import os_helper @@ -82,21 +83,8 @@ def data_file(*name): CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") CAFILE_CACERT = data_file("capath", "5ed36f99.0") -CERTFILE_INFO = { - 'issuer': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'notAfter': 'Jan 24 04:21:36 2043 GMT', - 'notBefore': 'Nov 25 04:21:36 2023 GMT', - 'serialNumber': '53E14833F7546C29256DD0F034F776C5E983004C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} +with open(data_file('keycert.pem.reference')) as file: + CERTFILE_INFO = literal_eval(file.read()) # empty CRL CRLFILE = data_file("revocation.crl") @@ -106,23 +94,8 @@ def data_file(*name): SINGED_CERTFILE_ONLY = data_file("cert3.pem") SIGNED_CERTFILE_HOSTNAME = 'localhost' -SIGNED_CERTFILE_INFO = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} +with open(data_file('keycert3.pem.reference')) as file: + SIGNED_CERTFILE_INFO = literal_eval(file.read()) SIGNED_CERTFILE2 = data_file("keycert4.pem") SIGNED_CERTFILE2_HOSTNAME = 'fakehostname' @@ -410,6 +383,7 @@ def test_random(self): ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) def test_parse_cert(self): + self.maxDiff = None # note that this uses an 'unofficial' function in _ssl.c, # provided solely for this test, to exercise the certificate # parsing code diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index d16ad7ef5d4328..b14d500a9c6e97 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -805,6 +805,7 @@ def test_windows_feature_macros(self): "PyUnicode_DecodeUnicodeEscape", "PyUnicode_EncodeFSDefault", "PyUnicode_EncodeLocale", + "PyUnicode_Equal", "PyUnicode_EqualToUTF8", "PyUnicode_EqualToUTF8AndSize", "PyUnicode_FSConverter", diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 6600dcf9157971..4de6c1cba152bd 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -1665,7 +1665,7 @@ def test_startswith_endswith_errors(self): self.assertIn('str', exc) self.assertIn('tuple', exc) - @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') def test_format_float(self): # should not format with a comma, but always with C locale self.assertEqual('1.0', '%.1f' % 1.0) diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 038746e26c24ad..09f6f656bfcb0d 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -5,13 +5,22 @@ import locale import re import os +import platform import sys from test import support -from test.support import skip_if_buggy_ucrt_strfptime, warnings_helper +from test.support import warnings_helper +from test.support import skip_if_buggy_ucrt_strfptime, run_with_locales from datetime import date as datetime_date import _strptime +libc_ver = platform.libc_ver() +if libc_ver[0] == 'glibc': + glibc_ver = tuple(map(int, libc_ver[1].split('.'))) +else: + glibc_ver = None + + class getlang_Tests(unittest.TestCase): """Test _getlang""" def test_basic(self): @@ -207,8 +216,8 @@ class StrptimeTests(unittest.TestCase): """Tests for _strptime.strptime.""" def setUp(self): - """Create testing time tuple.""" - self.time_tuple = time.gmtime() + """Create testing time tuples.""" + self.time_tuple = time.localtime() def test_ValueError(self): # Make sure ValueError is raised when match fails or format is bad @@ -283,60 +292,73 @@ def test_strptime_exception_context(self): # additional check for IndexError branch (issue #19545) with self.assertRaises(ValueError) as e: _strptime._strptime_time('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + self.assertIsNone(e.exception.__context__) def test_unconverteddata(self): # Check ValueError is raised when there is unconverted data self.assertRaises(ValueError, _strptime._strptime_time, "10 12", "%m") - def helper(self, directive, position): + def roundtrip(self, fmt, position, time_tuple=None): """Helper fxn in testing.""" - fmt = "%d %Y" if directive == 'd' else "%" + directive - strf_output = time.strftime(fmt, self.time_tuple) + if time_tuple is None: + time_tuple = self.time_tuple + strf_output = time.strftime(fmt, time_tuple) strp_output = _strptime._strptime_time(strf_output, fmt) - self.assertTrue(strp_output[position] == self.time_tuple[position], - "testing of '%s' directive failed; '%s' -> %s != %s" % - (directive, strf_output, strp_output[position], - self.time_tuple[position])) + self.assertEqual(strp_output[position], time_tuple[position], + "testing of %r format failed; %r -> %r != %r" % + (fmt, strf_output, strp_output[position], + time_tuple[position])) + if support.verbose >= 3: + print("testing of %r format: %r -> %r" % + (fmt, strf_output, strp_output[position])) def test_year(self): # Test that the year is handled properly - for directive in ('y', 'Y'): - self.helper(directive, 0) + self.roundtrip('%Y', 0) + self.roundtrip('%y', 0) + self.roundtrip('%Y', 0, (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + # Must also make sure %y values are correct for bounds set by Open Group - for century, bounds in ((1900, ('69', '99')), (2000, ('00', '68'))): - for bound in bounds: - strp_output = _strptime._strptime_time(bound, '%y') - expected_result = century + int(bound) - self.assertTrue(strp_output[0] == expected_result, - "'y' test failed; passed in '%s' " - "and returned '%s'" % (bound, strp_output[0])) + strptime = _strptime._strptime_time + self.assertEqual(strptime('00', '%y')[0], 2000) + self.assertEqual(strptime('68', '%y')[0], 2068) + self.assertEqual(strptime('69', '%y')[0], 1969) + self.assertEqual(strptime('99', '%y')[0], 1999) def test_month(self): # Test for month directives - for directive in ('B', 'b', 'm'): - self.helper(directive, 1) + self.roundtrip('%m', 1) + + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', '') + def test_month_locale(self): + # Test for month directives + self.roundtrip('%B', 1) + self.roundtrip('%b', 1) + for m in range(1, 13): + self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) def test_day(self): # Test for day directives - self.helper('d', 2) + self.roundtrip('%d %Y', 2) def test_hour(self): # Test hour directives - self.helper('H', 3) - strf_output = time.strftime("%I %p", self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%I %p") - self.assertTrue(strp_output[3] == self.time_tuple[3], - "testing of '%%I %%p' directive failed; '%s' -> %s != %s" % - (strf_output, strp_output[3], self.time_tuple[3])) + self.roundtrip('%H', 3) + + # NB: Only works on locales with AM/PM + @run_with_locales('LC_TIME', 'C', 'en_US', 'ja_JP') + def test_hour_locale(self): + # Test hour directives + self.roundtrip('%I %p', 3) def test_minute(self): # Test minute directives - self.helper('M', 4) + self.roundtrip('%M', 4) def test_second(self): # Test second directives - self.helper('S', 5) + self.roundtrip('%S', 5) def test_fraction(self): # Test microseconds @@ -347,12 +369,18 @@ def test_fraction(self): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w', 'u'): - self.helper(directive,6) + self.roundtrip('%w', 6) + self.roundtrip('%u', 6) + + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', '') + def test_weekday_locale(self): + # Test weekday directives + self.roundtrip('%A', 6) + self.roundtrip('%a', 6) def test_julian(self): # Test julian directives - self.helper('j', 7) + self.roundtrip('%j', 7) def test_offset(self): one_hour = 60 * 60 @@ -449,20 +477,112 @@ def test_bad_timezone(self): "time.daylight set to %s and passing in %s" % (time.tzname, tz_value, time.daylight, tz_name)) - def test_date_time(self): + # NB: Does not roundtrip in some locales due to the ambiguity of + # the date and time representation (bugs in locales?): + # * Seconds are not included: bem_ZM, bokmal, ff_SN, nb_NO, nn_NO, + # no_NO, norwegian, nynorsk. + # * Hours are in 12-hour notation without AM/PM indication: hy_AM, + # id_ID, ms_MY. + # * Year is not included: ha_NG. + # * Use non-Gregorian calendar: lo_LA, thai, th_TH. + # On Windows: ar_IN, ar_SA, fa_IR, ps_AF. + # + # BUG: Generates regexp that does not match the current date and time + # for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG', + 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', + 'my_MM', 'or_IN', 'shn_MM', 'az_IR') + def test_date_time_locale(self): # Test %c directive - for position in range(6): - self.helper('c', position) - - def test_date(self): + loc = locale.getlocale(locale.LC_TIME)[0] + if glibc_ver and glibc_ver < (2, 31) and loc == 'br_FR': + self.skipTest('%c in locale br_FR does not include time') + now = time.time() + self.roundtrip('%c', slice(0, 6), time.localtime(now)) + # 1 hour 20 minutes 30 seconds ago + self.roundtrip('%c', slice(0, 6), time.localtime(now - 4830)) + # 12 hours ago + self.roundtrip('%c', slice(0, 6), time.localtime(now - 12*3600)) + # different days of the week + for i in range(1, 7): + self.roundtrip('%c', slice(0, 6), time.localtime(now - i*24*3600)) + # different months + for i in range(1, 12): + self.roundtrip('%c', slice(0, 6), time.localtime(now - i*30*24*3600)) + # different year + self.roundtrip('%c', slice(0, 6), time.localtime(now - 366*24*3600)) + + # NB: Dates before 1969 do not roundtrip on some locales: + # az_IR, bo_CN, bo_IN, dz_BT, eu_ES, eu_FR, fa_IR, or_IN. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG', + 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', + 'my_MM', 'shn_MM') + def test_date_time_locale2(self): + # Test %c directive + self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%c', slice(0, 6), (1800, 1, 1, 0, 0, 0, 0, 1, 0)) + + # NB: Does not roundtrip because use non-Gregorian calendar: + # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF. + # BUG: Generates regexp that does not match the current date + # for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'eu_ES', 'ar_AE', + 'az_IR', 'my_MM', 'or_IN', 'shn_MM') + def test_date_locale(self): # Test %x directive - for position in range(0,3): - self.helper('x', position) - - def test_time(self): + now = time.time() + self.roundtrip('%x', slice(0, 3), time.localtime(now)) + # different days of the week + for i in range(1, 7): + self.roundtrip('%x', slice(0, 3), time.localtime(now - i*24*3600)) + # different months + for i in range(1, 12): + self.roundtrip('%x', slice(0, 3), time.localtime(now - i*30*24*3600)) + # different year + self.roundtrip('%x', slice(0, 3), time.localtime(now - 366*24*3600)) + + # NB: Dates before 1969 do not roundtrip on many locales, including C. + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM') + def test_date_locale2(self): + # Test %x directive + self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%x', slice(0, 3), (1800, 1, 1, 0, 0, 0, 0, 1, 0)) + + # NB: Does not roundtrip in some locales due to the ambiguity of + # the time representation (bugs in locales?): + # * Seconds are not included: bokmal, ff_SN, nb_NO, nn_NO, no_NO, + # norwegian, nynorsk. + # * Hours are in 12-hour notation without AM/PM indication: hy_AM, + # ms_MY, sm_WS. + # BUG: Generates regexp that does not match the current time for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET', + 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO', + 'ti_ET', 'tig_ER', 'wal_ET') + def test_time_locale(self): # Test %X directive - for position in range(3,6): - self.helper('X', position) + loc = locale.getlocale(locale.LC_TIME)[0] + pos = slice(3, 6) + if glibc_ver and glibc_ver < (2, 29) and loc in { + 'aa_ET', 'am_ET', 'byn_ER', 'gez_ET', 'om_ET', + 'sid_ET', 'so_SO', 'ti_ET', 'tig_ER', 'wal_ET'}: + # Hours are in 12-hour notation without AM/PM indication. + # Ignore hours. + pos = slice(4, 6) + now = time.time() + self.roundtrip('%X', pos, time.localtime(now)) + # 1 hour 20 minutes 30 seconds ago + self.roundtrip('%X', pos, time.localtime(now - 4830)) + # 12 hours ago + self.roundtrip('%X', pos, time.localtime(now - 12*3600)) def test_percent(self): # Make sure % signs are handled properly @@ -714,12 +834,7 @@ def test_new_localetime(self): def test_TimeRE_recreation_locale(self): # The TimeRE instance should be recreated upon changing the locale. - locale_info = locale.getlocale(locale.LC_TIME) - try: - locale.setlocale(locale.LC_TIME, ('en_US', 'UTF8')) - except locale.Error: - self.skipTest('test needs en_US.UTF8 locale') - try: + with support.run_with_locale('LC_TIME', 'en_US.UTF8'): _strptime._strptime_time('10 2004', '%d %Y') # Get id of current cache object. first_time_re = _strptime._TimeRE_cache @@ -736,10 +851,6 @@ def test_TimeRE_recreation_locale(self): # to the resetting to the original locale. except locale.Error: self.skipTest('test needs de_DE.UTF8 locale') - # Make sure we don't trample on the locale setting once we leave the - # test. - finally: - locale.setlocale(locale.LC_TIME, locale_info) @support.run_with_tz('STD-1DST,M4.1.0,M10.1.0') def test_TimeRE_recreation_timezone(self): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index bdbf8800cfd8f6..04ec3ed0837c82 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,4 +1,5 @@ from collections import abc +from itertools import combinations import array import gc import math @@ -11,12 +12,22 @@ from test import support from test.support import import_helper, suppress_immortalization from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin ISBIGENDIAN = sys.byteorder == "big" integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +INF = float('inf') +NAN = float('nan') + +try: + struct.pack('C', 1j) + have_c_complex = True +except struct.error: + have_c_complex = False + def iter_integer_formats(byteorders=byteorders): for code in integer_codes: for byteorder in byteorders: @@ -33,7 +44,7 @@ def bigendian_to_native(value): else: return string_reverse(value) -class StructTest(unittest.TestCase): +class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_isbigendian(self): self.assertEqual((struct.pack('=i', 1)[0] == 0), ISBIGENDIAN) @@ -529,6 +540,9 @@ def __bool__(self): for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: self.assertTrue(struct.unpack('>?', c)[0]) + self.assertTrue(struct.unpack('E', '>C', ' 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index de0e0b430a21bf..75710db7d05375 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1919,6 +1919,26 @@ def test_roundtrip(self): self.check_roundtrip(r"f'\\\\N{{'") self.check_roundtrip(r"f'\\\\\\N{{'") self.check_roundtrip(r"f'\\\\\\\\N{{'") + + self.check_roundtrip(r"f'\n{{foo}}'") + self.check_roundtrip(r"f'\\n{{foo}}'") + self.check_roundtrip(r"f'\\\n{{foo}}'") + self.check_roundtrip(r"f'\\\\n{{foo}}'") + + self.check_roundtrip(r"f'\t{{foo}}'") + self.check_roundtrip(r"f'\\t{{foo}}'") + self.check_roundtrip(r"f'\\\t{{foo}}'") + self.check_roundtrip(r"f'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\t{{foo}}'") + self.check_roundtrip(r"rf'\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\{{foo}}'") + self.check_roundtrip(r"f'\\{{foo}}'") + self.check_roundtrip(r"rf'\\\{{foo}}'") + self.check_roundtrip(r"f'\\\\{{foo}}'") cases = [ """ if 1: diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index 72446267f04759..d2ef59a29ca350 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -39,6 +39,15 @@ def test_invalid_char_quotes(self): tomllib.loads("v = '\n'") self.assertTrue(" '\\n' " in str(exc_info.exception)) + def test_type_error(self): + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(b"v = 1") # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bytes'") + + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(False) # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") + def test_module_name(self): self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 455fea034198a6..77ef0c5b3c480d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -150,7 +150,7 @@ def test_no_caret_with_no_debug_ranges_flag_python_traceback(self): import traceback try: x = 1 / 0 - except: + except ZeroDivisionError: traceback.print_exc() """) try: @@ -550,9 +550,10 @@ class PurePythonExceptionFormattingMixin: def get_exception(self, callable, slice_start=0, slice_end=-1): try: callable() - self.fail("No exception thrown.") - except: + except BaseException: return traceback.format_exc().splitlines()[slice_start:slice_end] + else: + self.fail("No exception thrown.") callable_line = get_exception.__code__.co_firstlineno + 2 @@ -2237,7 +2238,7 @@ def test_context_suppression(self): try: try: raise Exception - except: + except Exception: raise ZeroDivisionError from None except ZeroDivisionError as _: e = _ @@ -2589,9 +2590,9 @@ def exc(): try: try: raise EG("eg1", [ValueError(1), TypeError(2)]) - except: + except EG: raise EG("eg2", [ValueError(3), TypeError(4)]) - except: + except EG: raise ImportError(5) expected = ( @@ -2641,7 +2642,7 @@ def exc(): except Exception as e: exc = e raise EG("eg", [VE(1), exc, VE(4)]) - except: + except EG: raise EG("top", [VE(5)]) expected = (f' + Exception Group Traceback (most recent call last):\n' @@ -3454,7 +3455,7 @@ def test_long_context_chain(self): def f(): try: 1/0 - except: + except ZeroDivisionError: f() try: @@ -3558,7 +3559,7 @@ def test_comparison_params_variations(self): def raise_exc(): try: raise ValueError('bad value') - except: + except ValueError: raise def raise_with_locals(): diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 9a04a95dc40d65..eeaf5de2e303f6 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -227,13 +227,13 @@ def test_element_create_image(self): foreground='blue', background='yellow') img3 = tkinter.BitmapImage(master=self.root, file=imgfile, foreground='white', background='black') - style.element_create('Button.button', 'image', + style.element_create('TestButton.button', 'image', img1, ('pressed', img2), ('active', img3), border=(2, 4), sticky='we') - self.assertIn('Button.button', style.element_names()) + self.assertIn('TestButton.button', style.element_names()) - style.layout('Button', [('Button.button', {'sticky': 'news'})]) - b = ttk.Button(self.root, style='Button') + style.layout('TestButton', [('TestButton.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='TestButton') b.pack(expand=True, fill='both') self.assertEqual(b.winfo_reqwidth(), 16) self.assertEqual(b.winfo_reqheight(), 16) diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index cb210b7d2fc960..10bec33be617a1 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -336,7 +336,8 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', ) - IDENTIFY_AS = 'Entry.field' if sys.platform == 'darwin' else 'textarea' + # bpo-27313: macOS Tk/Tcl may or may not report 'Entry.field'. + IDENTIFY_AS = {'Entry.field', 'textarea'} def setUp(self): super().setUp() @@ -373,8 +374,7 @@ def test_identify(self): self.entry.pack() self.entry.update() - # bpo-27313: macOS Cocoa widget differs from X, allow either - self.assertEqual(self.entry.identify(5, 5), self.IDENTIFY_AS) + self.assertIn(self.entry.identify(5, 5), self.IDENTIFY_AS) self.assertEqual(self.entry.identify(-1, -1), "") self.assertRaises(tkinter.TclError, self.entry.identify, None, 5) @@ -461,7 +461,7 @@ class ComboboxTest(EntryTest, unittest.TestCase): 'validate', 'validatecommand', 'values', 'width', 'xscrollcommand', ) - IDENTIFY_AS = 'Combobox.button' if sys.platform == 'darwin' else 'textarea' + IDENTIFY_AS = {'Combobox.button', 'textarea'} def setUp(self): super().setUp() @@ -963,8 +963,7 @@ def create(self, **kwargs): return ttk.Scrollbar(self.root, **kwargs) -@add_standard_options(PixelSizeTests if tk_version >= (8, 7) else IntegerSizeTests, - StandardTtkOptionsTests) +@add_standard_options(StandardTtkOptionsTests) class NotebookTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'padding', 'style', 'takefocus', 'width', @@ -983,6 +982,20 @@ def setUp(self): def create(self, **kwargs): return ttk.Notebook(self.root, **kwargs) + def test_configure_height(self): + widget = self.create() + if get_tk_patchlevel(self.root) < (8, 6, 15): + self.checkIntegerParam(widget, 'height', 402, -402, 0) + else: + self.checkPixelsParam(widget, 'height', '10c', 402, -402, 0, conv=False) + + def test_configure_width(self): + widget = self.create() + if get_tk_patchlevel(self.root) < (8, 6, 15): + self.checkIntegerParam(widget, 'width', 402, -402, 0) + else: + self.checkPixelsParam(widget, 'width', '10c', 402, -402, 0, conv=False) + def test_tab_identifiers(self): self.nb.forget(0) self.nb.hide(self.child2) @@ -1191,7 +1204,7 @@ class SpinboxTest(EntryTest, unittest.TestCase): 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', 'width', 'wrap', 'xscrollcommand', ) - IDENTIFY_AS = 'Spinbox.field' if sys.platform == 'darwin' else 'textarea' + IDENTIFY_AS = {'Spinbox.field', 'textarea'} def setUp(self): super().setUp() diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 49d6aa810304fb..230bbe646baf28 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -4,7 +4,9 @@ from test.support import check_syntax_error, run_code from test.typinganndata import mod_generics_cache -from typing import Callable, TypeAliasType, TypeVar, get_args +from typing import ( + Callable, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args, +) class TypeParamsInvalidTest(unittest.TestCase): @@ -211,6 +213,59 @@ def test_generic(self): self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] + + def test_type_params_order_with_defaults(self): + HasNoDefaultT = TypeVar("HasNoDefaultT") + WithDefaultT = TypeVar("WithDefaultT", default=int) + + HasNoDefaultP = ParamSpec("HasNoDefaultP") + WithDefaultP = ParamSpec("WithDefaultP", default=HasNoDefaultP) + + HasNoDefaultTT = TypeVarTuple("HasNoDefaultTT") + WithDefaultTT = TypeVarTuple("WithDefaultTT", default=HasNoDefaultTT) + + for type_params in [ + (HasNoDefaultT, WithDefaultT), + (HasNoDefaultP, WithDefaultP), + (HasNoDefaultTT, WithDefaultTT), + ]: + with self.subTest(type_params=type_params): + TypeAliasType("A", int, type_params=type_params) # ok + + msg = "follows default type parameter" + for type_params in [ + (WithDefaultT, HasNoDefaultT), + (WithDefaultP, HasNoDefaultP), + (WithDefaultTT, HasNoDefaultTT), + (WithDefaultT, HasNoDefaultP), # different types + ]: + with self.subTest(type_params=type_params): + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=type_params) + + def test_expects_type_like(self): + T = TypeVar("T") + + msg = "Expected a type param" + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1,)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1, 2)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(T, 2)) def test_keywords(self): TA = TypeAliasType(name="TA", value=int) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 91082e6b23c04b..257b7fa95dcb76 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -375,7 +375,7 @@ class X: with self.assertRaises(NotImplementedError): annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): - annotate(annotationlib.Format.SOURCE) + annotate(annotationlib.Format.STRING) with self.assertRaises(NotImplementedError): annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index dc0c0d0829f8d3..433b19593bdd04 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1440,7 +1440,7 @@ def f[T: int = int, **P = int, *Ts = int](): pass self.assertIs(case(1), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) - self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.SOURCE), 'int') + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') def test_constraints(self): def f[T: (int, str)](): pass @@ -1451,4 +1451,15 @@ def f[T: (int, str)](): pass self.assertEqual(case.evaluate_constraints(1), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) - self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.SOURCE), '(int, str)') + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') + + def test_const_evaluator(self): + T = TypeVar("T", bound=int) + self.assertEqual(repr(T.evaluate_bound), ">") + + ConstEvaluator = type(T.evaluate_bound) + + with self.assertRaisesRegex(TypeError, r"cannot create '_typing\._ConstEvaluator' instances"): + ConstEvaluator() # This used to segfault. + with self.assertRaisesRegex(TypeError, r"cannot set 'attribute' attribute of immutable type '_typing\._ConstEvaluator'"): + ConstEvaluator.attribute = 1 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3c9e33e3c9dbfc..d1161719d98040 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -398,7 +398,7 @@ def test(i, format_spec, result): test(123456, "1=20", '11111111111111123456') test(123456, "*=20", '**************123456') - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_float__format__locale(self): # test locale support for __format__ code 'n' @@ -407,7 +407,7 @@ def test_float__format__locale(self): self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n')) self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n')) - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_int__format__locale(self): # test locale support for __format__ code 'n' for integers diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3ac6b97383fcef..2f1f9e86a0bce4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7059,7 +7059,7 @@ class C: self.assertIsInstance(annos['x'], annotationlib.ForwardRef) self.assertEqual(annos['x'].__arg__, 'undefined') - self.assertEqual(get_type_hints(C, format=annotationlib.Format.SOURCE), + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), {'x': 'undefined'}) @@ -7898,7 +7898,7 @@ class Z(NamedTuple): self.assertEqual(Z.__annotations__, annos) self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Z.__annotate__(annotationlib.Format.SOURCE), {"a": "None", "b": "str"}) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) def test_future_annotations(self): code = """ @@ -8241,7 +8241,7 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__annotations__, annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Emp.__annotate__(annotationlib.Format.SOURCE), {'name': 'str', 'id': 'int'}) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8603,7 +8603,7 @@ class A[T](TypedDict): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8616,7 +8616,7 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8628,7 +8628,7 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) self.assertEqual(A2.__annotations__, {'a': T}) - self.assertEqual(A2.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8640,7 +8640,7 @@ class B(A[KT], total=False): self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) - self.assertEqual(B.__annotate__(annotationlib.Format.SOURCE), {'a': 'T', 'b': 'KT'}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8665,7 +8665,7 @@ class C(B[int]): 'b': KT, 'c': int, }) - self.assertEqual(C.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8689,7 +8689,7 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) - self.assertEqual(Point3D.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'T', 'c': 'KT', @@ -8725,7 +8725,7 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) - self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8929,7 +8929,7 @@ class A(TypedDict): A.__annotations__ self.assertEqual( - A.__annotate__(annotationlib.Format.SOURCE), + A.__annotate__(annotationlib.Format.STRING), {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', 'z': 'Required[undefined]'}, ) diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index c9c20f008ca5a2..f260769eb8c35e 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -8,8 +8,10 @@ Mock, ANY, _CallList, patch, PropertyMock, _callable ) +from dataclasses import dataclass, field, InitVar from datetime import datetime from functools import partial +from typing import ClassVar class SomeClass(object): def one(self, a, b): pass @@ -1034,6 +1036,76 @@ def f(a): pass self.assertEqual(mock.mock_calls, []) self.assertEqual(rv.mock_calls, []) + def test_dataclass_post_init(self): + @dataclass + class WithPostInit: + a: int = field(init=False) + b: int = field(init=False) + def __post_init__(self): + self.a = 1 + self.b = 2 + + for mock in [ + create_autospec(WithPostInit, instance=True), + create_autospec(WithPostInit()), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + # Classes do not have these fields: + mock = create_autospec(WithPostInit) + msg = "Mock object has no attribute" + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b + + def test_dataclass_default(self): + @dataclass + class WithDefault: + a: int + b: int = 0 + + for mock in [ + create_autospec(WithDefault, instance=True), + create_autospec(WithDefault(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + def test_dataclass_with_method(self): + @dataclass + class WithMethod: + a: int + def b(self) -> int: + return 1 + + for mock in [ + create_autospec(WithMethod, instance=True), + create_autospec(WithMethod(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + mock.b.assert_not_called() + + def test_dataclass_with_non_fields(self): + @dataclass + class WithNonFields: + a: ClassVar[int] + b: InitVar[int] + + msg = "Mock object has no attribute" + for mock in [ + create_autospec(WithNonFields, instance=True), + create_autospec(WithNonFields(1)), + ]: + with self.subTest(mock=mock): + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b class TestCallList(unittest.TestCase): diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index ef02c64f886f8a..8b4bb8750f8f5c 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -6,7 +6,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G, skip_on_s390x +from test.support import bigmemtest, _1G, _4G, is_s390x zlib = import_helper.import_module('zlib') @@ -33,8 +33,9 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() -# bpo-46623: On s390x, when a hardware accelerator is used, using different -# ways to compress data with zlib can produce different compressed data. +# bpo-46623: When a hardware accelerator is used (currently only on s390x), +# using different ways to compress data with zlib can produce different +# compressed data. # Simplified test_pair() code: # # def func1(data): @@ -57,8 +58,10 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): # # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # -# Make the assumption that s390x always has an accelerator to simplify the skip -# condition. +# To simplify the skip condition, make the assumption that s390x always has an +# accelerator, and nothing else has it. +HW_ACCELERATED = is_s390x + class VersionTestCase(unittest.TestCase): @@ -223,12 +226,14 @@ def test_keywords(self): bufsize=zlib.DEF_BUF_SIZE), HAMLET_SCENE) - @skip_on_s390x def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 x = zlib.compress(data) - self.assertEqual(zlib.compress(bytearray(data)), x) + # With hardware acceleration, the compressed bytes + # might not be identical. + if not HW_ACCELERATED: + self.assertEqual(zlib.compress(bytearray(data)), x) for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) @@ -275,7 +280,6 @@ def test_64bit_compress(self, size): class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Test compression object - @skip_on_s390x def test_pair(self): # straightforward compress/decompress objects datasrc = HAMLET_SCENE * 128 @@ -286,7 +290,10 @@ def test_pair(self): x1 = co.compress(data) x2 = co.flush() self.assertRaises(zlib.error, co.flush) # second flush should not work - self.assertEqual(x1 + x2, datazip) + # With hardware acceleration, the compressed bytes might not + # be identical. + if not HW_ACCELERATED: + self.assertEqual(x1 + x2, datazip) for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))): dco = zlib.decompressobj() y1 = dco.decompress(v1 + v2) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7f418bb7a1b37f..7ece4e9b70d31b 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -200,7 +200,7 @@ def escape_brackets(self, token): characters[-2::-1] ) ) - if n_backslashes % 2 == 0: + if n_backslashes % 2 == 0 or characters[-1] != "N": characters.append(character) else: consume_until_next_bracket = True @@ -510,7 +510,7 @@ def error(message, filename=None, location=None): sys.exit(1) # Parse the arguments and options - parser = argparse.ArgumentParser(prog='python -m tokenize') + parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 45ca7a89630f0e..5671326646ca5a 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -71,7 +71,12 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n # The spec allows converting "\r\n" to "\n", even in string # literals. Let's do so to simplify parsing. - src = s.replace("\r\n", "\n") + try: + src = s.replace("\r\n", "\n") + except (AttributeError, TypeError): + raise TypeError( + f"Expected str object, not '{type(s).__qualname__}'" + ) from None pos = 0 out = Output(NestedDict(), Flags()) header: Key = () diff --git a/Lib/typing.py b/Lib/typing.py index 9377e771d60f4b..c924c767042552 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -242,21 +242,10 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - # When changing this function, don't forget about - # `_collections_abc._type_repr`, which does the same thing - # and must be consistent with this one. - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: - return '...' - if isinstance(obj, types.FunctionType): - return obj.__name__ if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return repr(obj) + return annotationlib.value_to_string(obj) def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): @@ -1047,7 +1036,7 @@ def evaluate_forward_ref( * Recursively evaluates forward references nested within the type hint. * Rejects certain objects that are not valid type hints. * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *SOURCE* formats. + * Supports the *FORWARDREF* and *STRING* formats. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -1064,7 +1053,7 @@ def evaluate_forward_ref( if type_params is _sentinel: _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") type_params = () - if format == annotationlib.Format.SOURCE: + if format == annotationlib.Format.STRING: return forward_ref.__forward_arg__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref @@ -2391,7 +2380,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, hints = {} for base in reversed(obj.__mro__): ann = annotationlib.get_annotations(base, format=format) - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: hints.update(ann) continue if globalns is None: @@ -2415,7 +2404,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, value = _eval_type(value, base_globals, base_locals, base.__type_params__, format=format, owner=obj) hints[name] = value - if include_extras or format is annotationlib.Format.SOURCE: + if include_extras or format is annotationlib.Format.STRING: return hints else: return {k: _strip_annotations(t) for k, t in hints.items()} @@ -2429,7 +2418,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, and not hasattr(obj, '__annotate__') ): raise TypeError(f"{obj!r} is not a module, class, or callable.") - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: return hints if globalns is None: @@ -2948,14 +2937,10 @@ def annotate(format): if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): return checked_types else: - return _convert_to_source(types) + return annotationlib.annotations_to_string(types) return annotate -def _convert_to_source(types): - return {n: t if isinstance(t, str) else _type_repr(t) for n, t in types.items()} - - # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', @@ -2987,7 +2972,7 @@ def __new__(cls, typename, bases, ns): def annotate(format): annos = annotationlib.call_annotate_function(original_annotate, format) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: return {key: _type_check(val, f"field {key} annotation must be a type") for key, val in annos.items()} return annos @@ -3235,13 +3220,13 @@ def __annotate__(format): annos.update(base_annos) if own_annotate is not None: own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: own = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own.items() } - elif format == annotationlib.Format.SOURCE: - own = _convert_to_source(own_annotations) + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) else: own = own_checked_annotations annos.update(own) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index bb34c7436047ad..21ca061a77c26f 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -34,6 +34,7 @@ import pkgutil from inspect import iscoroutinefunction import threading +from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -628,7 +629,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None, *, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -2218,7 +2221,7 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() - def reset_mock(self, /, *args, return_value=False, **kwargs): + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): if ( return_value and self._mock_name @@ -2754,7 +2757,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) - _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + _kwargs = {'spec': [f.name for f in dataclass_fields]} + else: + _kwargs = {'spec': spec} + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2811,7 +2822,7 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, _name='()', _parent=mock, wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2825,10 +2836,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue child_kwargs = {'spec': original} # Wrap child attributes also. diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 028e9483196694..a5d348ba4cf121 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -303,8 +303,11 @@ def setup_python(self, context): copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', - f'python3.{sys.version_info[1]}'): + + suffixes = ['python', 'python3', f'python3.{sys.version_info[1]}'] + if sys.version_info[:2] == (3, 14): + suffixes.append('𝜋thon') + for suffix in suffixes: path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -572,8 +575,7 @@ def create(env_dir, system_site_packages=False, clear=False, def main(args=None): import argparse - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' + parser = argparse.ArgumentParser(description='Creates virtual Python ' 'environments in one or ' 'more target ' 'directories.', diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index cbd4873f012246..4593799b7e9b0e 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -14,8 +14,9 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # Call hash to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then diff --git a/Lib/weakref.py b/Lib/weakref.py index 25b70927e29c31..94e4278143c987 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -19,7 +19,7 @@ ReferenceType, _remove_dead_weakref) -from _weakrefset import WeakSet, _IterationGuard +from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. import sys @@ -105,34 +105,14 @@ def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(wr.key) - else: - # Atomic removal is necessary since this function - # can be called asynchronously by the GC - _atomic_removal(self.data, wr.key) + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(self.data, wr.key) self._remove = remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() self.data = {} self.update(other, **kw) - def _commit_removals(self, _atomic_removal=_remove_dead_weakref): - pop = self._pending_removals.pop - d = self.data - # We shouldn't encounter any KeyError, because this method should - # always be called *before* mutating the dict. - while True: - try: - key = pop() - except IndexError: - return - _atomic_removal(d, key) - def __getitem__(self, key): - if self._pending_removals: - self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) @@ -140,18 +120,12 @@ def __getitem__(self, key): return o def __delitem__(self, key): - if self._pending_removals: - self._commit_removals() del self.data[key] def __len__(self): - if self._pending_removals: - self._commit_removals() return len(self.data) def __contains__(self, key): - if self._pending_removals: - self._commit_removals() try: o = self.data[key]() except KeyError: @@ -162,38 +136,28 @@ def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): - if self._pending_removals: - self._commit_removals() new = WeakValueDictionary() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy - if self._pending_removals: - self._commit_removals() new = self.__class__() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): - if self._pending_removals: - self._commit_removals() try: wr = self.data[key] except KeyError: @@ -207,21 +171,15 @@ def get(self, key, default=None): return o def items(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - v = wr() - if v is not None: - yield k, v + for k, wr in self.data.copy().items(): + v = wr() + if v is not None: + yield k, v def keys(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - if wr() is not None: - yield k + for k, wr in self.data.copy().items(): + if wr() is not None: + yield k __iter__ = keys @@ -235,23 +193,15 @@ def itervaluerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - yield from self.data.values() + yield from self.data.copy().values() def values(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for wr in self.data.values(): - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy().values(): + obj = wr() + if obj is not None: + yield obj def popitem(self): - if self._pending_removals: - self._commit_removals() while True: key, wr = self.data.popitem() o = wr() @@ -259,8 +209,6 @@ def popitem(self): return key, o def pop(self, key, *args): - if self._pending_removals: - self._commit_removals() try: o = self.data.pop(key)() except KeyError: @@ -279,16 +227,12 @@ def setdefault(self, key, default=None): except KeyError: o = None if o is None: - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(self, other=None, /, **kwargs): - if self._pending_removals: - self._commit_removals() d = self.data if other is not None: if not hasattr(other, "items"): @@ -308,9 +252,7 @@ def valuerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - return list(self.data.values()) + return list(self.data.copy().values()) def __ior__(self, other): self.update(other) @@ -369,57 +311,22 @@ def __init__(self, dict=None): def remove(k, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(k) - else: - try: - del self.data[k] - except KeyError: - pass + try: + del self.data[k] + except KeyError: + pass self._remove = remove - # A list of dead weakrefs (keys to be removed) - self._pending_removals = [] - self._iterating = set() - self._dirty_len = False if dict is not None: self.update(dict) - def _commit_removals(self): - # NOTE: We don't need to call this method before mutating the dict, - # because a dead weakref never compares equal to a live weakref, - # even if they happened to refer to equal objects. - # However, it means keys may already have been removed. - pop = self._pending_removals.pop - d = self.data - while True: - try: - key = pop() - except IndexError: - return - - try: - del d[key] - except KeyError: - pass - - def _scrub_removals(self): - d = self.data - self._pending_removals = [k for k in self._pending_removals if k in d] - self._dirty_len = False - def __delitem__(self, key): - self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): - if self._dirty_len and self._pending_removals: - # self._pending_removals may still contain keys which were - # explicitly removed, we have to scrub them (see issue #21173). - self._scrub_removals() - return len(self.data) - len(self._pending_removals) + return len(self.data) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) @@ -429,11 +336,10 @@ def __setitem__(self, key, value): def copy(self): new = WeakKeyDictionary() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -441,11 +347,10 @@ def copy(self): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -459,26 +364,23 @@ def __contains__(self, key): return wr in self.data def items(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - key = wr() - if key is not None: - yield key, value + for wr, value in self.data.copy().items(): + key = wr() + if key is not None: + yield key, value def keys(self): - with _IterationGuard(self): - for wr in self.data: - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy(): + obj = wr() + if obj is not None: + yield obj __iter__ = keys def values(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - if wr() is not None: - yield value + for wr, value in self.data.copy().items(): + if wr() is not None: + yield value def keyrefs(self): """Return a list of weak references to the keys. @@ -493,7 +395,6 @@ def keyrefs(self): return list(self.data) def popitem(self): - self._dirty_len = True while True: key, value = self.data.popitem() o = key() @@ -501,7 +402,6 @@ def popitem(self): return o, value def pop(self, key, *args): - self._dirty_len = True return self.data.pop(ref(key), *args) def setdefault(self, key, default=None): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b97738836d92cc..f5f0ed44884142 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.14' - tcl_checksum='5880225babf7954c58d4fb0f5cf6279104ce1cd6aa9b71e9a6322540e1c4de66' + tcl_tk_ver='8.6.15' + tcl_checksum='861e159753f2e2fbd6ec1484103715b0be56be3357522b858d3cbb5f893ffef1' - tk_checksum='8ffdb720f47a6ca6107eac2dd877e30b0ef7fac14f3a84ebbd0b3612cee41a94' + tk_checksum='550969f35379f952b3020f3ab7b9dd5bfd11c1ef7c9b7c6a75f5c49aca793fec' tk_patches = [] diff --git a/Makefile.pre.in b/Makefile.pre.in index a4d99262702a17..07c8a4d20142db 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -490,7 +490,7 @@ PYTHON_OBJS= \ Python/thread.o \ Python/traceback.o \ Python/tracemalloc.o \ - Python/typeid.o \ + Python/uniqueid.o \ Python/getopt.o \ Python/pystrcmp.o \ Python/pystrtod.o \ @@ -1279,7 +1279,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ - $(srcdir)/Include/internal/pycore_typeid.h \ + $(srcdir)/Include/internal/pycore_uniqueid.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ diff --git a/Misc/ACKS b/Misc/ACKS index ef0f403950255b..a1769d9601a2ea 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1552,7 +1552,9 @@ Lisa Roach Carl Robben Ben Roberts Mark Roberts +Tony Roberts Andy Robinson +Izan "TizzySaurus" Robinson Jim Robinson Yolanda Robla Daniel Rocco @@ -1812,6 +1814,7 @@ Reuben Sumner Eryk Sun Sanjay Sundaresan Marek Šuppa +Danica J. Sutherland Hisao Suzuki Kalle Svensson Andrew Svetlov diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index bd002b6ad3db9b..3e82de9ef266d6 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -226,8 +226,8 @@ thread at the time the function is called. .. section: Core and Builtins Enable ``from __future__ import annotations`` (:pep:`563`) by default. The -values found in :attr:`__annotations__` dicts are now strings, e.g. ``{"x": -"int"}`` instead of ``{"x": int}``. +values found in :attr:`~object.__annotations__` dicts are now strings, for +example ``{"x": "int"}`` instead of ``{"x": int}``. .. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 954f5c18b48000..5418d5d59dd583 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -486,8 +486,8 @@ Use ``dis.Positions`` in ``dis.Instruction`` instead of a regular ``tuple``. .. nonce: geS-aP .. section: Library -:mod:`pdb` now gracefully handles ``help`` when :attr:`__doc__` is missing, -for example when run with pregenerated optimized ``.pyc`` files. +:mod:`pdb` now gracefully handles ``help`` when :attr:`~module.__doc__` is +missing, for example when run with pregenerated optimized ``.pyc`` files. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 7e0f86179bce50..f2668e99a6299b 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -4237,8 +4237,8 @@ by :mod:`asyncio` to AIX platform only. .. nonce: 4dzB80 .. section: Library -Set :attr:`doctest.DocTest.lineno` to ``None`` when object does not have -:attr:`__doc__`. +Set :attr:`doctest.DocTest.lineno` to ``None`` when an object does not have +:attr:`~definition.__doc__`. .. diff --git a/Misc/NEWS.d/next/Build/2024-05-22-13-18-02.gh-issue-119400.WEt83v.rst b/Misc/NEWS.d/next/Build/2024-05-22-13-18-02.gh-issue-119400.WEt83v.rst new file mode 100644 index 00000000000000..b4029f205797e4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-22-13-18-02.gh-issue-119400.WEt83v.rst @@ -0,0 +1,2 @@ +``make_ssl_certs``, the script that prepares certificate data for the +test suite, now allows specifying expiration dates. diff --git a/Misc/NEWS.d/next/Build/2024-10-13-21-11-30.gh-issue-125269.BC-fdo.rst b/Misc/NEWS.d/next/Build/2024-10-13-21-11-30.gh-issue-125269.BC-fdo.rst new file mode 100644 index 00000000000000..24f5469e8a664b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-13-21-11-30.gh-issue-125269.BC-fdo.rst @@ -0,0 +1,2 @@ +Fix detection of whether ``-latomic`` is needed when cross-compiling CPython +using the configure script. diff --git a/Misc/NEWS.d/next/C API/2018-06-30-21-48-16.bpo-34008.2Wjtm0.rst b/Misc/NEWS.d/next/C API/2018-06-30-21-48-16.bpo-34008.2Wjtm0.rst new file mode 100644 index 00000000000000..1a01dafc758004 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-06-30-21-48-16.bpo-34008.2Wjtm0.rst @@ -0,0 +1,2 @@ +Added ``Py_IsInitialized`` to the list of APIs that are safe to call before +the interpreter is initialized, and updated the embedding tests to cover it. diff --git a/Misc/NEWS.d/next/C API/2022-08-05-19-41-20.gh-issue-87135.SCNBYj.rst b/Misc/NEWS.d/next/C API/2022-08-05-19-41-20.gh-issue-87135.SCNBYj.rst new file mode 100644 index 00000000000000..6387d69bc267c6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-08-05-19-41-20.gh-issue-87135.SCNBYj.rst @@ -0,0 +1,15 @@ +Attempting to acquire the GIL after runtime finalization has begun in a +different thread now causes the thread to hang rather than terminate, which +avoids potential crashes or memory corruption caused by attempting to +terminate a thread that is running code not specifically designed to support +termination. In most cases this hanging is harmless since the process will +soon exit anyway. + +The ``PyThread_exit_thread`` function is now deprecated. Its behavior is +inconsistent across platforms, and it can only be used safely in the +unlikely case that every function in the entire call stack has been designed +to support the platform-dependent termination mechanism. It is recommended +that users of this function change their design to not require thread +termination. In the unlikely case that thread termination is needed and can +be done safely, users may migrate to calling platform-specific APIs such as +``pthread_exit`` (POSIX) or ``_endthreadex`` (Windows) directly. diff --git a/Misc/NEWS.d/next/C_API/2024-09-24-20-34-21.gh-issue-124296.S4QoS1.rst b/Misc/NEWS.d/next/C_API/2024-09-24-20-34-21.gh-issue-124296.S4QoS1.rst new file mode 100644 index 00000000000000..e7b9187655eb31 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-24-20-34-21.gh-issue-124296.S4QoS1.rst @@ -0,0 +1,3 @@ +:c:type:`PyDictObject` no longer maintains a private version tag field +``ma_version_tag`` per :pep:`699`. This field was originally added in +Python 3.6 (:pep:`509`) and deprecated in Python 3.12. diff --git a/Misc/NEWS.d/next/C_API/2024-09-25-11-44-02.gh-issue-124502.qWuDjT.rst b/Misc/NEWS.d/next/C_API/2024-09-25-11-44-02.gh-issue-124502.qWuDjT.rst new file mode 100644 index 00000000000000..f515619328b359 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-25-11-44-02.gh-issue-124502.qWuDjT.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnicode_Equal` function to the limited C API: test if two +strings are equal. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2024-10-03-14-06-08.gh-issue-123961.uwJQTY.rst b/Misc/NEWS.d/next/C_API/2024-10-03-14-06-08.gh-issue-123961.uwJQTY.rst new file mode 100644 index 00000000000000..40c26e15a2de92 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-10-03-14-06-08.gh-issue-123961.uwJQTY.rst @@ -0,0 +1,3 @@ +Convert the :ref:`curses.window ` static type exposed +by the :c:macro:`!PyCursesWindow_Type` macro in ``Include/py_curses.h`` to a +:ref:`heap type `. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst index 1e5ad7d08eed7c..567c096472878f 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst @@ -1,3 +1,3 @@ :func:`classmethod` and :func:`staticmethod` now wrap the -:attr:`__annotations__` and :attr:`!__annotate__` attributes of their -underlying callable lazily. See :pep:`649`. Patch by Jelle Zijlstra. +:attr:`~object.__annotations__` and :attr:`~object.__annotate__` attributes of +their underlying callable lazily. See :pep:`649`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-30-11-41-35.gh-issue-122445.Rq0bjS.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-30-11-41-35.gh-issue-122445.Rq0bjS.rst index f5aa07c6513ea9..cb9dabbc71706f 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-07-30-11-41-35.gh-issue-122445.Rq0bjS.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-30-11-41-35.gh-issue-122445.Rq0bjS.rst @@ -1 +1 @@ -Add only fields which are modified via self.* to :attr:`~class.__static_attributes__`. +Add only fields which are modified via self.* to :attr:`~type.__static_attributes__`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst new file mode 100644 index 00000000000000..a6dec66a743f92 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst @@ -0,0 +1,2 @@ +Fix bug where ``bool(a)`` can be invoked more than once during the +evaluation of a compound boolean expression. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-10-03-22-26-39.gh-issue-124871.tAMF47.rst b/Misc/NEWS.d/next/Core and Builtins/2024-10-03-22-26-39.gh-issue-124871.tAMF47.rst new file mode 100644 index 00000000000000..185cb3048fadf5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-10-03-22-26-39.gh-issue-124871.tAMF47.rst @@ -0,0 +1,2 @@ +Fix compiler bug (in some versions of 3.13) where an assertion fails during reachability +analysis. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst new file mode 100644 index 00000000000000..de62875e16475d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst @@ -0,0 +1,2 @@ +Improved JIT memory consumption by periodically freeing memory used by infrequently-executed code. +This change is especially likely to improve the memory footprint of long-running programs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst new file mode 100644 index 00000000000000..25b47d5fbaefa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst @@ -0,0 +1,3 @@ +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-16-57-34.gh-issue-119726.DseseK.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-16-57-34.gh-issue-119726.DseseK.rst new file mode 100644 index 00000000000000..c01eeff952534f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-19-16-57-34.gh-issue-119726.DseseK.rst @@ -0,0 +1,2 @@ +The JIT now generates more efficient code for calls to C functions resulting +in up to 0.8% memory savings and 1.5% speed improvement on AArch64. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst new file mode 100644 index 00000000000000..b5f423f3ff1c96 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst @@ -0,0 +1,2 @@ +Fix PyREPL failure when a keyboard interrupt is triggered after using a +history search diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst new file mode 100644 index 00000000000000..58e79f22ac0f90 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst @@ -0,0 +1,2 @@ +Fix nondeterminism in compilation by sorting the value of +:attr:`~type.__static_attributes__`. Patch by kp2pml30. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-14-45-56.gh-issue-124513.ywiXtr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-14-45-56.gh-issue-124513.ywiXtr.rst new file mode 100644 index 00000000000000..691e03b3b98e7a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-14-45-56.gh-issue-124513.ywiXtr.rst @@ -0,0 +1,2 @@ +Fix a crash in FrameLocalsProxy constructor: check the number of arguments. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst new file mode 100644 index 00000000000000..1005c651849f45 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst @@ -0,0 +1,3 @@ +When deallocating an object with inline values whose ``__dict__`` is still +live: if memory allocation for the inline values fails, clear the +dictionary. Prevents an interpreter crash. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst new file mode 100644 index 00000000000000..4cdbb205c962c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst @@ -0,0 +1,2 @@ +The ``__main__`` module no longer always contains an ``__annotations__`` +dictionary in its global namespace. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst new file mode 100644 index 00000000000000..fc3f8af72d87bf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst @@ -0,0 +1,3 @@ +Fix a bug that can cause a crash when sub-interpreters use "basic" +single-phase extension modules. Shared objects could refer to PyGC_Head +nodes that had been freed as part of interpreter cleanup. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst new file mode 100644 index 00000000000000..e3741321006548 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst @@ -0,0 +1,5 @@ +Fix a crash caused by immortal interned strings being shared between +sub-interpreters that use basic single-phase init. In that case, the string +can be used by an interpreter that outlives the interpreter that created and +interned it. For interpreters that share obmalloc state, also share the +interned dict with the main interpreter. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-17-18-53.gh-issue-124642.OCjhBJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-17-18-53.gh-issue-124642.OCjhBJ.rst new file mode 100644 index 00000000000000..29763844a9f592 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-17-18-53.gh-issue-124642.OCjhBJ.rst @@ -0,0 +1 @@ +Fixed scalability issue in free-threaded builds for lock-free reads from dictionaries in multi-threaded scenarios diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-30-16-39-37.gh-issue-118093.J2A3gz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-30-16-39-37.gh-issue-118093.J2A3gz.rst new file mode 100644 index 00000000000000..2e5c64581b6aef --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-30-16-39-37.gh-issue-118093.J2A3gz.rst @@ -0,0 +1,2 @@ +Improve the experimental JIT compiler's ability to stay "on trace" when +encountering highly-biased branches. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst new file mode 100644 index 00000000000000..5cd34535d674d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst @@ -0,0 +1,3 @@ +Fix a crash in the :meth:`~object.__str__` method of :exc:`UnicodeError` +objects when the :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values are invalid or out-of-range. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst new file mode 100644 index 00000000000000..8971e052860225 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst @@ -0,0 +1,2 @@ +Fix :func:`tokenize.untokenize` producing invalid syntax for +double braces preceded by certain escape characters. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-07-23-33-18.gh-issue-125039.MKTyNI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-07-23-33-18.gh-issue-125039.MKTyNI.rst new file mode 100644 index 00000000000000..93716c0b1c0df1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-07-23-33-18.gh-issue-125039.MKTyNI.rst @@ -0,0 +1 @@ +Make ``this_instr`` and ``prev_instr`` const in cases generator. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-10-12-04-56.gh-issue-125174._8h6T7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-10-12-04-56.gh-issue-125174._8h6T7.rst new file mode 100644 index 00000000000000..c7eaac32601bb3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-10-12-04-56.gh-issue-125174._8h6T7.rst @@ -0,0 +1,4 @@ +Make the handling of reference counts of immortal objects more robust. +Immortal objects with reference counts that deviate from their original +reference count by up to a billion (half a billion on 32 bit builds) are +still counted as immortal. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-10-14-47-13.gh-issue-125221.nfSQzT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-10-14-47-13.gh-issue-125221.nfSQzT.rst new file mode 100644 index 00000000000000..c79650c3a64feb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-10-14-47-13.gh-issue-125221.nfSQzT.rst @@ -0,0 +1,2 @@ +Fix possible race condition when calling :meth:`~object.__reduce_ex__` for the +first time in the free threading build. diff --git a/Misc/NEWS.d/next/Documentation/2018-07-04-20-35-25.bpo-34008.bqecIb.rst b/Misc/NEWS.d/next/Documentation/2018-07-04-20-35-25.bpo-34008.bqecIb.rst new file mode 100644 index 00000000000000..a89086af35bfc1 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2018-07-04-20-35-25.bpo-34008.bqecIb.rst @@ -0,0 +1,8 @@ +The :c:func:`Py_Main` documentation moved from the "Very High Level API" section to the +"Initialization and Finalization" section. + +Also make it explicit that we expect ``Py_Main`` to typically be called instead +of ``Py_Initialize`` rather than after it (since ``Py_Main`` makes its own +call to ``Py_Initialize``). Document that calling both is +supported but is version dependent on which settings +will be applied correctly. diff --git a/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst b/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst new file mode 100644 index 00000000000000..60f75ae0c21326 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst @@ -0,0 +1,2 @@ +Writers of CPython's documentation can now use ``next`` as the version for +the ``versionchanged``, ``versionadded``, ``deprecated`` directives. diff --git a/Misc/NEWS.d/next/Documentation/2024-08-01-17-18-21.gh-issue-70870.fZnBM9.rst b/Misc/NEWS.d/next/Documentation/2024-08-01-17-18-21.gh-issue-70870.fZnBM9.rst new file mode 100644 index 00000000000000..ba607bf7a42cd9 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-08-01-17-18-21.gh-issue-70870.fZnBM9.rst @@ -0,0 +1,3 @@ +Clarified the dual usage of the term "free variable" (both the formal +meaning of any reference to names defined outside the local scope, and the +narrower pragmatic meaning of nonlocal variables named in ``co_freevars``). diff --git a/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst new file mode 100644 index 00000000000000..f9da7b8a5724f5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst @@ -0,0 +1,2 @@ +Remove coverity scan from the CPython repo. It has not been used since 2020 +and is currently unmaintained. diff --git a/Misc/NEWS.d/next/Documentation/2024-09-27-16-47-48.gh-issue-124720.nVSTVb.rst b/Misc/NEWS.d/next/Documentation/2024-09-27-16-47-48.gh-issue-124720.nVSTVb.rst new file mode 100644 index 00000000000000..6bef1e4158400b --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-27-16-47-48.gh-issue-124720.nVSTVb.rst @@ -0,0 +1,2 @@ +Update "Using Python on a Mac" section of the "Python Setup and Usage" +document and include information on installing free-threading support. diff --git a/Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst b/Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst new file mode 100644 index 00000000000000..e910da5b879ba5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst @@ -0,0 +1,4 @@ +The :mod:`importlib.metadata` documentation now includes semantic +cross-reference targets for the significant documented APIs. This means +intersphinx references like :func:`importlib.metadata.version` will +now work as expected. diff --git a/Misc/NEWS.d/next/Documentation/2024-10-10-02-56-24.gh-issue-124872.0mDDOq.rst b/Misc/NEWS.d/next/Documentation/2024-10-10-02-56-24.gh-issue-124872.0mDDOq.rst new file mode 100644 index 00000000000000..69a5c764d05a2e --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-10-10-02-56-24.gh-issue-124872.0mDDOq.rst @@ -0,0 +1,3 @@ +Added definitions for :term:`context`, :term:`current context`, and +:term:`context management protocol`, updated related definitions to be +consistent, and expanded the documentation for :class:`contextvars.Context`. diff --git a/Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst b/Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst new file mode 100644 index 00000000000000..541f6212794ef2 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst @@ -0,0 +1,2 @@ +Increase currently inadequate vertical spacing for the IDLE browsers (path, +module, and stack) on high-resolution monitors. diff --git a/Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst b/Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst new file mode 100644 index 00000000000000..221c8e05fa98aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` metavar processing to allow positional arguments to have a +tuple metavar. diff --git a/Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst b/Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst new file mode 100644 index 00000000000000..9610fa90ef0a98 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst @@ -0,0 +1 @@ +Do not translate user-provided strings in :class:`argparse.ArgumentParser`. diff --git a/Misc/NEWS.d/next/Library/2023-06-16-14-52-00.gh-issue-102450.MfeR6A.rst b/Misc/NEWS.d/next/Library/2023-06-16-14-52-00.gh-issue-102450.MfeR6A.rst new file mode 100644 index 00000000000000..abfad5fa63b777 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-16-14-52-00.gh-issue-102450.MfeR6A.rst @@ -0,0 +1,2 @@ +Add missing ISO-8601 24:00 alternative to midnight of next day to :meth:`datetime.datetime.fromisoformat` and :meth:`datetime.time.fromisoformat`. +Patch by Izan "TizzySaurus" Robinson (tizzysaurus@gmail.com) diff --git a/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst new file mode 100644 index 00000000000000..8e1937ab73c31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst @@ -0,0 +1,9 @@ +Add *doc* parameter to :func:`dataclasses.field`, so it can be stored and +shown as a documentation / metadata. If ``@dataclass(slots=True)`` is used, +then the supplied string is availabl in the :attr:`~object.__slots__` dict. +Otherwise, the supplied string is only available in the corresponding +:class:`dataclasses.Field` object. + +In order to support this feature we are changing the ``__slots__`` format +in dataclasses from :class:`tuple` to :class:`dict` +when documentation / metadata is present. diff --git a/Misc/NEWS.d/next/Library/2024-02-27-10-22-15.gh-issue-115937.0cVNur.rst b/Misc/NEWS.d/next/Library/2024-02-27-10-22-15.gh-issue-115937.0cVNur.rst new file mode 100644 index 00000000000000..f9dae0c9b8e2b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-27-10-22-15.gh-issue-115937.0cVNur.rst @@ -0,0 +1,3 @@ +Removed extra preprocessing for the ``__signature__`` attribute: the code +just check if it's a :class:`inspect.Signature` instance. Patch by Sergey B +Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst b/Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst new file mode 100644 index 00000000000000..d090f931f0238d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst @@ -0,0 +1 @@ +Always use :func:`str` to print ``choices`` in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst b/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst new file mode 100644 index 00000000000000..e47e2ae89dbff0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst @@ -0,0 +1,2 @@ +Positional arguments of :func:`functools.partial` objects +now support placeholders via :data:`functools.Placeholder`. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst b/Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst new file mode 100644 index 00000000000000..33ef1c91591c54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst @@ -0,0 +1 @@ +Added ``commands`` argument to :func:`pdb.set_trace` which allows users to send debugger commands from the source file. diff --git a/Misc/NEWS.d/next/Library/2024-06-15-23-38-36.gh-issue-120284.HwsAtY.rst b/Misc/NEWS.d/next/Library/2024-06-15-23-38-36.gh-issue-120284.HwsAtY.rst new file mode 100644 index 00000000000000..a2a6883c3d7686 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-15-23-38-36.gh-issue-120284.HwsAtY.rst @@ -0,0 +1,2 @@ +Allow :meth:`asyncio.Runner.run` to accept :term:`awaitable` +objects instead of simply :term:`coroutine`\s. diff --git a/Misc/NEWS.d/next/Library/2024-06-19-19-53-42.gh-issue-41431.gnkUc5.rst b/Misc/NEWS.d/next/Library/2024-06-19-19-53-42.gh-issue-41431.gnkUc5.rst new file mode 100644 index 00000000000000..18e3506a60c455 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-19-19-53-42.gh-issue-41431.gnkUc5.rst @@ -0,0 +1,2 @@ +Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +Contributed by Wannes Boeykens. diff --git a/Misc/NEWS.d/next/Library/2024-07-10-08-13-34.gh-issue-121249.W9Gd09.rst b/Misc/NEWS.d/next/Library/2024-07-10-08-13-34.gh-issue-121249.W9Gd09.rst new file mode 100644 index 00000000000000..2d41fca45bcad9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-10-08-13-34.gh-issue-121249.W9Gd09.rst @@ -0,0 +1,3 @@ +Support the :c:expr:`float complex` and :c:expr:`double complex` +C types in the :mod:`struct` module if the compiler has C11 complex +arithmetic. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-07-15-19-25-25.gh-issue-121798.GmuBDu.rst b/Misc/NEWS.d/next/Library/2024-07-15-19-25-25.gh-issue-121798.GmuBDu.rst new file mode 100644 index 00000000000000..5706e4bffeb4a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-15-19-25-25.gh-issue-121798.GmuBDu.rst @@ -0,0 +1,2 @@ +Add alternative :class:`~decimal.Decimal` constructor +:meth:`Decimal.from_number() `. diff --git a/Misc/NEWS.d/next/Library/2024-07-15-19-34-56.gh-issue-121797.qDqj59.rst b/Misc/NEWS.d/next/Library/2024-07-15-19-34-56.gh-issue-121797.qDqj59.rst new file mode 100644 index 00000000000000..9525379587f6cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-15-19-34-56.gh-issue-121797.qDqj59.rst @@ -0,0 +1,2 @@ +Add alternative :class:`~fractions.Fraction` constructor +:meth:`Fraction.from_number() `. diff --git a/Misc/NEWS.d/next/Library/2024-08-06-07-24-00.gh-issue-118974.qamsCQ.rst b/Misc/NEWS.d/next/Library/2024-08-06-07-24-00.gh-issue-118974.qamsCQ.rst new file mode 100644 index 00000000000000..79480a69c1a90e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-06-07-24-00.gh-issue-118974.qamsCQ.rst @@ -0,0 +1,2 @@ +Add ``decorator`` parameter to :func:`dataclasses.make_dataclass` +to customize the functional creation of dataclasses. diff --git a/Misc/NEWS.d/next/Library/2024-08-23-15-49-10.gh-issue-116810.QLBUU8.rst b/Misc/NEWS.d/next/Library/2024-08-23-15-49-10.gh-issue-116810.QLBUU8.rst new file mode 100644 index 00000000000000..0e5256e7151c5a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-23-15-49-10.gh-issue-116810.QLBUU8.rst @@ -0,0 +1,4 @@ +Resolve a memory leak introduced in CPython 3.10's :mod:`ssl` when the +:attr:`ssl.SSLSocket.session` property was accessed. Speeds up read and +write access to said property by no longer unnecessarily cloning session +objects via serialization. diff --git a/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst new file mode 100644 index 00000000000000..e388541f1c2c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst @@ -0,0 +1,4 @@ +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. diff --git a/Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst b/Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst new file mode 100644 index 00000000000000..258dd591fce767 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst @@ -0,0 +1 @@ +Added a new argument ``mode`` to :class:`pdb.Pdb`. Only allow :mod:`pdb` from command line to use ``restart`` command. diff --git a/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst new file mode 100644 index 00000000000000..a4428e20f3ccdd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst @@ -0,0 +1,5 @@ +The default :mod:`multiprocessing` start method on Linux and other POSIX +systems has been changed away from often unsafe ``"fork"`` to ``"forkserver"`` +(when the platform supports sending file handles over pipes as most do) or +``"spawn"``. Mac and Windows are unchanged as they already default to +``"spawn"``. diff --git a/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst new file mode 100644 index 00000000000000..707c4d651cb5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst @@ -0,0 +1,2 @@ +Fix disallowing abbreviation of single-dash long options in :mod:`argparse` +with ``allow_abbrev=False``. diff --git a/Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst b/Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst new file mode 100644 index 00000000000000..7a389fefc6c54b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst @@ -0,0 +1,3 @@ +Modify dataclasses to support zero-argument super() when ``slots=True`` is +specified. This works by modifying all references to ``__class__`` to point +to the newly created class. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-00-01-24.gh-issue-124400.0XCgfe.rst b/Misc/NEWS.d/next/Library/2024-09-24-00-01-24.gh-issue-124400.0XCgfe.rst new file mode 100644 index 00000000000000..25ee01e3108bf8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-00-01-24.gh-issue-124400.0XCgfe.rst @@ -0,0 +1 @@ +Fixed a :mod:`pdb` bug where ``until`` has no effect when it appears in a ``commands`` sequence. Also avoid printing the frame information at a breakpoint that has a command list containing a command that resumes execution. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst new file mode 100644 index 00000000000000..dff902d8c6139a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst @@ -0,0 +1,2 @@ +:mod:`argparse` vim supports abbreviated single-dash long options separated +by ``=`` from its value. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst new file mode 100644 index 00000000000000..38c030668b6b42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst @@ -0,0 +1,4 @@ +Add support for :func:`dataclasses.dataclass` in +:func:`unittest.mock.create_autospec`. Now ``create_autospec`` will check +for potential dataclasses and use :func:`dataclasses.fields` function to +retrieve the spec information. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst b/Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst new file mode 100644 index 00000000000000..53dbabd9480ddb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst @@ -0,0 +1,3 @@ +:func:`os.pidfd_open` and :func:`signal.pidfd_send_signal` are now +unavailable when building against Android API levels older than 31, since +the underlying system calls may cause a crash. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst new file mode 100644 index 00000000000000..45fe4786fa6563 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst @@ -0,0 +1,2 @@ +Due to unreliable results on some devices, :func:`time.strftime` no longer +accepts negative years on Android. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst b/Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst new file mode 100644 index 00000000000000..55f1d4b41125c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst @@ -0,0 +1,4 @@ +Fixed bug in itertools.tee() handling of other tee inputs (a tee in a tee). +The output now has the promised *n* independent new iterators. Formerly, +the first iterator was identical (not independent) to the input iterator. +This would sometimes give surprising results. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst new file mode 100644 index 00000000000000..20ba1534f5e99d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst @@ -0,0 +1,4 @@ +Fix support of arguments with :ref:`choices` in :mod:`argparse`. Positional +arguments with :ref:`nargs` equal to ``'?'`` or ``'*'`` no longer check +:ref:`default` against ``choices``. Optional arguments with ``nargs`` equal +to ``'?'`` no longer check :ref:`const` against ``choices``. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst new file mode 100644 index 00000000000000..4dbf4eb709733d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst @@ -0,0 +1,2 @@ +Fix :class:`typing.TypeAliasType` not to be generic, when ``type_params`` is +an empty tuple. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst new file mode 100644 index 00000000000000..1a8c1427b6b9b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst @@ -0,0 +1,2 @@ +Fix a crash related to an integer overflow in :func:`curses.resizeterm` +and :func:`curses.resize_term`. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst new file mode 100644 index 00000000000000..bb451cdd9ae44c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` support of positional arguments with ``nargs='?'``, +``default=argparse.SUPPRESS`` and specified ``type``. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst new file mode 100644 index 00000000000000..33ae037ae56b0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst @@ -0,0 +1 @@ +Fixed crash when using :func:`gc.get_referents` on a capsule object. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-22-06-52.gh-issue-124552.1nQKNM.rst b/Misc/NEWS.d/next/Library/2024-09-25-22-06-52.gh-issue-124552.1nQKNM.rst new file mode 100644 index 00000000000000..39dde4c774ba5d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-22-06-52.gh-issue-124552.1nQKNM.rst @@ -0,0 +1 @@ +Improve the accuracy of :mod:`bdb`'s check for the possibility of breakpoint in a frame. This makes it possible to disable unnecessary events in functions. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-00-35-24.gh-issue-116750.X1aMHI.rst b/Misc/NEWS.d/next/Library/2024-09-26-00-35-24.gh-issue-116750.X1aMHI.rst new file mode 100644 index 00000000000000..cf9dacf4007c28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-00-35-24.gh-issue-116750.X1aMHI.rst @@ -0,0 +1 @@ +Provide :func:`sys.monitoring.clear_tool_id` to unregister all events and callbacks set by the tool. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst new file mode 100644 index 00000000000000..801a5fdd4abd4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst @@ -0,0 +1,2 @@ +Fix support of :ref:`choices` with string value in :mod:`argparse`. Substrings +of the specified string no longer considered valid values. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst b/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst new file mode 100644 index 00000000000000..ac48bd84930745 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst @@ -0,0 +1 @@ +All :mod:`asyncio` REPL prompts run in the same :class:`context `. Contributed by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-22-14-12.gh-issue-58573.hozbm9.rst b/Misc/NEWS.d/next/Library/2024-09-26-22-14-12.gh-issue-58573.hozbm9.rst new file mode 100644 index 00000000000000..37d64ee536ff49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-22-14-12.gh-issue-58573.hozbm9.rst @@ -0,0 +1,2 @@ +Fix conflicts between abbreviated long options in the parent parser and +subparsers in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-06-39-32.gh-issue-101552.xYkzag.rst b/Misc/NEWS.d/next/Library/2024-09-27-06-39-32.gh-issue-101552.xYkzag.rst new file mode 100644 index 00000000000000..913a84de5fe6a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-06-39-32.gh-issue-101552.xYkzag.rst @@ -0,0 +1,4 @@ +Add an *annoation_format* parameter to :func:`inspect.signature`. Add an +*quote_annotation_strings* parameter to :meth:`inspect.Signature.format`. Use the +new functionality to improve the display of annotations in signatures in +:mod:`pydoc`. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst new file mode 100644 index 00000000000000..62639a16c52aa0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` for namespaces with not directly writable dict (e.g. +classes). diff --git a/Misc/NEWS.d/next/Library/2024-09-30-19-59-28.gh-issue-66436.4gYN_n.rst b/Misc/NEWS.d/next/Library/2024-09-30-19-59-28.gh-issue-66436.4gYN_n.rst new file mode 100644 index 00000000000000..69a77b01902873 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-30-19-59-28.gh-issue-66436.4gYN_n.rst @@ -0,0 +1,4 @@ +Improved :ref:`prog` default value for :class:`argparse.ArgumentParser`. It +will now include the name of the Python executable along with the module or +package name, or the path to a directory, ZIP file, or directory within a +ZIP file if the code was run that way. diff --git a/Misc/NEWS.d/next/Library/2024-09-30-20-46-32.gh-issue-124787.3FnJnP.rst b/Misc/NEWS.d/next/Library/2024-09-30-20-46-32.gh-issue-124787.3FnJnP.rst new file mode 100644 index 00000000000000..d9d1bbcf5a2fe4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-30-20-46-32.gh-issue-124787.3FnJnP.rst @@ -0,0 +1,4 @@ +Fix :class:`typing.TypeAliasType` with incorrect ``type_params`` argument. +Now it raises a :exc:`TypeError` when a type parameter without a default +follows one with a default, and when an entry in the ``type_params`` tuple +is not a type parameter object. diff --git a/Misc/NEWS.d/next/Library/2024-10-01-02-31-13.gh-issue-124693.qzbXKB.rst b/Misc/NEWS.d/next/Library/2024-10-01-02-31-13.gh-issue-124693.qzbXKB.rst new file mode 100644 index 00000000000000..3e87eb457d9911 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-01-02-31-13.gh-issue-124693.qzbXKB.rst @@ -0,0 +1 @@ +Fix a bug where :mod:`argparse` doesn't recognize negative complex numbers or negative numbers using scientific notation. diff --git a/Misc/NEWS.d/next/Library/2024-10-01-12-43-42.gh-issue-124835.SVyp3K.rst b/Misc/NEWS.d/next/Library/2024-10-01-12-43-42.gh-issue-124835.SVyp3K.rst new file mode 100644 index 00000000000000..09e5a046f83e49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-01-12-43-42.gh-issue-124835.SVyp3K.rst @@ -0,0 +1,3 @@ +Make :func:`tomllib.loads` raise :exc:`TypeError` not :exc:`AttributeError` +on bad input types that do not have the ``replace`` attribute. Improve error +message when :class:`bytes` is received. diff --git a/Misc/NEWS.d/next/Library/2024-10-01-13-11-53.gh-issue-85935.CTwJUy.rst b/Misc/NEWS.d/next/Library/2024-10-01-13-11-53.gh-issue-85935.CTwJUy.rst new file mode 100644 index 00000000000000..553f206bf26337 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-01-13-11-53.gh-issue-85935.CTwJUy.rst @@ -0,0 +1,4 @@ +:meth:`argparse.ArgumentParser.add_argument` now raises an exception if +an :ref:`action` that does not consume arguments (like 'store_const' or +'store_true') or explicit ``nargs=0`` are specified for positional +arguments. diff --git a/Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst b/Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst new file mode 100644 index 00000000000000..89610fa44bf743 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst @@ -0,0 +1 @@ +Fixed :exc:`AssertionError` when using :func:`!asyncio.staggered.staggered_race` with :attr:`asyncio.eager_task_factory`. diff --git a/Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst b/Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst new file mode 100644 index 00000000000000..c05d24a7c5aacb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst @@ -0,0 +1 @@ +Fix reference cycles left in tracebacks in :func:`asyncio.open_connection` when used with ``happy_eyeballs_delay`` diff --git a/Misc/NEWS.d/next/Library/2024-10-01-23-29-09.gh-issue-91818.Kz8cPI.rst b/Misc/NEWS.d/next/Library/2024-10-01-23-29-09.gh-issue-91818.Kz8cPI.rst new file mode 100644 index 00000000000000..f45f00e48a3830 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-01-23-29-09.gh-issue-91818.Kz8cPI.rst @@ -0,0 +1,4 @@ +The CLI of many modules (:mod:`ast`, :mod:`ensurepip`, :mod:`json`, +:mod:`pdb`, :mod:`sqlite3`, :mod:`tokenize`, :mod:`venv`) now uses the +actual executable name instead of simply "python" to display in the usage +message. diff --git a/Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst b/Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst new file mode 100644 index 00000000000000..6f5ad12d2c2981 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst @@ -0,0 +1,2 @@ +Fix detection of the minimal Queue API needed by the :mod:`logging` module. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-02-16-35-07.gh-issue-65865.S2D4wq.rst b/Misc/NEWS.d/next/Library/2024-10-02-16-35-07.gh-issue-65865.S2D4wq.rst new file mode 100644 index 00000000000000..106a8b81140520 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-16-35-07.gh-issue-65865.S2D4wq.rst @@ -0,0 +1,3 @@ +:mod:`argparse` now raises early error for invalid ``help`` arguments to +:meth:`~argparse.ArgumentParser.add_argument`, +:meth:`~argparse.ArgumentParser.add_subparsers` and :meth:`!add_parser`. diff --git a/Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst b/Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst new file mode 100644 index 00000000000000..f208793859bbf8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst @@ -0,0 +1,2 @@ +Allow calling :func:`os.path.exists` and :func:`os.path.lexists` with +keyword arguments on Windows. Fixes a regression in 3.13.0. diff --git a/Misc/NEWS.d/next/Library/2024-10-02-22-53-48.gh-issue-90102.4qX52R.rst b/Misc/NEWS.d/next/Library/2024-10-02-22-53-48.gh-issue-90102.4qX52R.rst new file mode 100644 index 00000000000000..0e708ed11d21b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-22-53-48.gh-issue-90102.4qX52R.rst @@ -0,0 +1,3 @@ +Skip the ``isatty`` system call during open() when the file is known to not +be a character device. This provides a slight performance improvement when +reading whole files. diff --git a/Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst b/Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst new file mode 100644 index 00000000000000..a7d6251f1e071f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-03-05-00-25.gh-issue-117151.Prdw_W.rst @@ -0,0 +1,3 @@ +The default buffer size used by :func:`shutil.copyfileobj` has been +increased from 64k to 256k on non-Windows platforms. It was already larger +on Windows. diff --git a/Misc/NEWS.d/next/Library/2024-10-03-17-13-22.gh-issue-124944.YyLAzf.rst b/Misc/NEWS.d/next/Library/2024-10-03-17-13-22.gh-issue-124944.YyLAzf.rst new file mode 100644 index 00000000000000..66af712c5ae5a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-03-17-13-22.gh-issue-124944.YyLAzf.rst @@ -0,0 +1 @@ +Add ``SO_ORIGINAL_DST`` to the :mod:`socket` module. diff --git a/Misc/NEWS.d/next/Library/2024-10-03-19-16-38.gh-issue-123961.ik1Dgs.rst b/Misc/NEWS.d/next/Library/2024-10-03-19-16-38.gh-issue-123961.ik1Dgs.rst new file mode 100644 index 00000000000000..b637b895d0b803 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-03-19-16-38.gh-issue-123961.ik1Dgs.rst @@ -0,0 +1,2 @@ +Convert :mod:`curses` to multi-phase initialization (:pep:`489`), thereby +fixing reference leaks at interpreter shutdown. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst b/Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst new file mode 100644 index 00000000000000..6895cffcf545fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst @@ -0,0 +1,5 @@ +Fix :func:`time.strptime` for ``%c`` and ``%x`` formats in many locales: +Arabic, Bislama, Breton, Bodo, Kashubian, Chuvash, Estonian, French, Irish, +Ge'ez, Gurajati, Manx Gaelic, Hebrew, Hindi, Chhattisgarhi, Haitian Kreyol, +Japanese, Kannada, Korean, Marathi, Malay, Norwegian, Nynorsk, Punjabi, +Rajasthani, Tok Pisin, Yoruba, Yue Chinese, Yau/Nungon and Chinese. diff --git a/Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst b/Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst new file mode 100644 index 00000000000000..534d5bb8c898da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst @@ -0,0 +1 @@ +Fix refcycles in exceptions raised from :class:`asyncio.TaskGroup` and the python implementation of :class:`asyncio.Future` diff --git a/Misc/NEWS.d/next/Library/2024-10-04-12-43-03.gh-issue-69998.DVqOXX.rst b/Misc/NEWS.d/next/Library/2024-10-04-12-43-03.gh-issue-69998.DVqOXX.rst new file mode 100644 index 00000000000000..65388e0b4e7ee4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-04-12-43-03.gh-issue-69998.DVqOXX.rst @@ -0,0 +1,3 @@ +Fix :func:`locale.nl_langinfo` in case when different categories have +different locales. The function now sets temporarily the ``LC_CTYPE`` locale +in some cases. This temporary change affects other threads. diff --git a/Misc/NEWS.d/next/Library/2024-10-05-15-49-53.gh-issue-124960.Bol9hT.rst b/Misc/NEWS.d/next/Library/2024-10-05-15-49-53.gh-issue-124960.Bol9hT.rst new file mode 100644 index 00000000000000..332d6bb54d80c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-05-15-49-53.gh-issue-124960.Bol9hT.rst @@ -0,0 +1 @@ +Fix support for the ``barry_as_FLUFL`` future flag in the new REPL. diff --git a/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst b/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst new file mode 100644 index 00000000000000..b5082b90721d42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst @@ -0,0 +1,3 @@ +Fix ``locale.nl_langinfo(locale.ALT_DIGITS)``. Now it returns a tuple of up +to 100 strings (an empty tuple on most locales). Previously it returned the +first item of that tuple or an empty string. diff --git a/Misc/NEWS.d/next/Library/2024-10-08-13-28-22.gh-issue-125096.Vz0W5g.rst b/Misc/NEWS.d/next/Library/2024-10-08-13-28-22.gh-issue-125096.Vz0W5g.rst new file mode 100644 index 00000000000000..c582a2dfe7243c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-08-13-28-22.gh-issue-125096.Vz0W5g.rst @@ -0,0 +1,5 @@ +If the :envvar:`PYTHON_BASIC_REPL` environment variable is set, the +:mod:`site` module no longer imports the :mod:`!_pyrepl` module. Moreover, +the :mod:`site` module now respects :option:`-E` and :option:`-I` command +line options: ignore :envvar:`PYTHON_BASIC_REPL` in this case. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst b/Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst new file mode 100644 index 00000000000000..9f1fd871e1d0b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst @@ -0,0 +1,4 @@ +Fix an issue where providing a :class:`pathlib.PurePath` object as an +initializer argument to a second :class:`~pathlib.PurePath` object with a +different :attr:`~pathlib.PurePath.parser` resulted in arguments to the +former object's initializer being joined by the latter object's parser. diff --git a/Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst b/Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst new file mode 100644 index 00000000000000..5d57cdbbbc2fe9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst @@ -0,0 +1 @@ +Don't copy arbitrary values to :c:expr:`_Bool` in the :mod:`struct` module. diff --git a/Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst b/Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst new file mode 100644 index 00000000000000..20f9c0b9c78b12 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst @@ -0,0 +1,4 @@ +Fix inheritance of nested mutually exclusive groups from parent parser in +:class:`argparse.ArgumentParser`. Previously, all nested mutually exclusive +groups lost their connection to the group containing them and were displayed +as belonging directly to the parser. diff --git a/Misc/NEWS.d/next/Library/2024-10-10-18-33-31.gh-issue-125235.0kOB5I.rst b/Misc/NEWS.d/next/Library/2024-10-10-18-33-31.gh-issue-125235.0kOB5I.rst new file mode 100644 index 00000000000000..f64d15917da1fc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-10-18-33-31.gh-issue-125235.0kOB5I.rst @@ -0,0 +1,2 @@ +Keep :mod:`tkinter` TCL paths in venv pointing to base installation on +Windows. diff --git a/Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst b/Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst new file mode 100644 index 00000000000000..abe37fefedc3be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst @@ -0,0 +1 @@ +Fix a bug where ArgumentError includes the incorrect ambiguous option in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-10-10-20-39-57.gh-issue-125243.eUbbtu.rst b/Misc/NEWS.d/next/Library/2024-10-10-20-39-57.gh-issue-125243.eUbbtu.rst new file mode 100644 index 00000000000000..49f84d9711819f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-10-20-39-57.gh-issue-125243.eUbbtu.rst @@ -0,0 +1,2 @@ +Fix data race when creating :class:`zoneinfo.ZoneInfo` objects in the free +threading build. diff --git a/Misc/NEWS.d/next/Library/2024-10-11-04-04-38.gh-issue-125260.PeZ0Mb.rst b/Misc/NEWS.d/next/Library/2024-10-11-04-04-38.gh-issue-125260.PeZ0Mb.rst new file mode 100644 index 00000000000000..fab524ea0185c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-11-04-04-38.gh-issue-125260.PeZ0Mb.rst @@ -0,0 +1,2 @@ +The :func:`gzip.compress` *mtime* parameter now defaults to 0 for reproducible output. +Patch by Bernhard M. Wiedemann and Adam Turner. diff --git a/Misc/NEWS.d/next/Library/2024-10-11-16-19-46.gh-issue-89967.vhWUOR.rst b/Misc/NEWS.d/next/Library/2024-10-11-16-19-46.gh-issue-89967.vhWUOR.rst new file mode 100644 index 00000000000000..d0860457c8e813 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-11-16-19-46.gh-issue-89967.vhWUOR.rst @@ -0,0 +1 @@ +Make :class:`~weakref.WeakKeyDictionary` and :class:`~weakref.WeakValueDictionary` safe against concurrent mutations from other threads. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst b/Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst new file mode 100644 index 00000000000000..cdfa8c191e8242 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst @@ -0,0 +1,2 @@ +Fix :func:`time.strptime` for ``%c``, ``%x`` and ``%X`` formats in many +locales that use non-ASCII digits, like Persian, Burmese, Odia and Shan. diff --git a/Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst b/Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst new file mode 100644 index 00000000000000..c890ecec8beaf8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst @@ -0,0 +1 @@ +Fixed the bug where :mod:`pdb` and :mod:`bdb` can step into the bottom caller frame. diff --git a/Misc/NEWS.d/next/Security/2024-10-09-20-08-13.gh-issue-125140.YgNWRB.rst b/Misc/NEWS.d/next/Security/2024-10-09-20-08-13.gh-issue-125140.YgNWRB.rst new file mode 100644 index 00000000000000..f4a49302372647 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-10-09-20-08-13.gh-issue-125140.YgNWRB.rst @@ -0,0 +1 @@ +Remove the current directory from ``sys.path`` when using PyREPL. diff --git a/Misc/NEWS.d/next/Tests/2023-08-03-17-26-55.gh-issue-107562.ZnbscS.rst b/Misc/NEWS.d/next/Tests/2023-08-03-17-26-55.gh-issue-107562.ZnbscS.rst new file mode 100644 index 00000000000000..cc368d4bdffbd2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-03-17-26-55.gh-issue-107562.ZnbscS.rst @@ -0,0 +1,3 @@ +Test certificates have been updated to expire far in the future. This allows +testing Y2038 with system time set to after that, so that actual Y2038 +issues can be exposed, and not masked by expired certificate errors. diff --git a/Misc/NEWS.d/next/Tests/2024-09-25-12-39-34.gh-issue-124378.Ywwgia.rst b/Misc/NEWS.d/next/Tests/2024-09-25-12-39-34.gh-issue-124378.Ywwgia.rst new file mode 100644 index 00000000000000..9ddcca0eb6036d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-09-25-12-39-34.gh-issue-124378.Ywwgia.rst @@ -0,0 +1 @@ +Updated ``test_ttk`` to pass with Tcl/Tk 8.6.15. diff --git a/Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst b/Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst new file mode 100644 index 00000000000000..c7181eb9c1f3a9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst @@ -0,0 +1,3 @@ +Re-enable skipped tests for :mod:`zlib` on the s390x architecture: only skip +checks of the compressed bytes, which can be different between zlib's +software implementation and the hardware-accelerated implementation. diff --git a/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst b/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst deleted file mode 100644 index 7f840b0556048a..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst +++ /dev/null @@ -1 +0,0 @@ -Updated bundled Tcl/Tk to 8.6.14. diff --git a/Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst b/Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst new file mode 100644 index 00000000000000..ca9845a8daea9d --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst @@ -0,0 +1 @@ +Updated bundled Tcl/Tk to 8.6.15. diff --git a/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst new file mode 100644 index 00000000000000..203868a8fee39c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst @@ -0,0 +1 @@ +Fix ``_Py_ThreadId`` for Windows builds using MinGW. Patch by Tony Roberts. diff --git a/Misc/NEWS.d/next/Windows/2024-09-27-15-07-30.gh-issue-124487.7LrwHC.rst b/Misc/NEWS.d/next/Windows/2024-09-27-15-07-30.gh-issue-124487.7LrwHC.rst new file mode 100644 index 00000000000000..93fb68d28c702e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-27-15-07-30.gh-issue-124487.7LrwHC.rst @@ -0,0 +1 @@ +Increases Windows required OS and API level to Windows 10. diff --git a/Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst b/Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst new file mode 100644 index 00000000000000..6d57aa1ee190d6 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst @@ -0,0 +1 @@ +Update bundled Tcl/Tk in macOS installer to 8.6.15. diff --git a/Misc/README b/Misc/README index 3dab768ba1a7a4..cbad9b72dc713c 100644 --- a/Misc/README +++ b/Misc/README @@ -17,7 +17,6 @@ python.man UNIX man page for the python interpreter python.pc.in Package configuration info template for pkg-config README The file you're reading now README.AIX Information about using Python on AIX -README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds svnmap.txt Map of old SVN revs and branches to hg changeset ids, diff --git a/Misc/README.coverity b/Misc/README.coverity deleted file mode 100644 index f5e1bf6f28d245..00000000000000 --- a/Misc/README.coverity +++ /dev/null @@ -1,22 +0,0 @@ - -Coverity has a static analysis tool (Prevent) which is similar to Klocwork. -They run their tool on the Python source code (SVN head) on a daily basis. -The results are available at: - - http://scan.coverity.com/ - -About 20 people have access to the analysis reports. Other -people can be added by request. - -Prevent was first run on the Python 2.5 source code in March 2006. -There were originally about 100 defects reported. Some of these -were false positives. Over 70 issues were uncovered. - -Each warning has a unique id and comments that can be made on it. -When checking in changes due to a warning, the unique id -as reported by the tool was added to the SVN commit message. - -False positives were annotated so that the comments can -be reviewed and reversed if the analysis was incorrect. - -Contact python-dev@python.org for more information. diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c deleted file mode 100644 index 90c72c7baa3f9e..00000000000000 --- a/Misc/coverity_model.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Coverity Scan model - * - * This is a modeling file for Coverity Scan. Modeling helps to avoid false - * positives. - * - * - A model file can't import any header files. - * - Therefore only some built-in primitives like int, char and void are - * available but not wchar_t, NULL etc. - * - Modeling doesn't need full structs and typedefs. Rudimentary structs - * and similar types are sufficient. - * - An uninitialized local pointer is not an error. It signifies that the - * variable could be either NULL or have some data. - * - * Coverity Scan doesn't pick up modifications automatically. The model file - * must be uploaded by an admin in the analysis settings of - * http://scan.coverity.com/projects/200 - */ - -/* dummy definitions, in most cases struct fields aren't required. */ - -#define NULL (void *)0 -#define assert(op) /* empty */ -typedef int sdigit; -typedef long Py_ssize_t; -typedef unsigned short wchar_t; -typedef struct {} PyObject; -typedef struct {} grammar; -typedef struct {} DIR; -typedef struct {} RFILE; - -/* Python/pythonrun.c - * resource leak false positive */ - -void Py_FatalError(const char *msg) { - __coverity_panic__(); -} - -/* Objects/longobject.c - * NEGATIVE_RETURNS false positive */ - -static PyObject *get_small_int(sdigit ival) -{ - /* Never returns NULL */ - PyObject *p; - assert(p != NULL); - return p; -} - -PyObject *PyLong_FromLong(long ival) -{ - PyObject *p; - int maybe; - - if ((ival >= -5) && (ival < 257 + 5)) { - p = get_small_int(ival); - assert(p != NULL); - return p; - } - if (maybe) - return p; - else - return NULL; -} - -PyObject *PyLong_FromLongLong(long long ival) -{ - return PyLong_FromLong((long)ival); -} - -PyObject *PyLong_FromSsize_t(Py_ssize_t ival) -{ - return PyLong_FromLong((long)ival); -} - -/* tainted sinks - * - * Coverity considers argv, environ, read() data etc as tainted. - */ - -PyObject *PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) -{ - __coverity_tainted_data_sink__(filename); - return NULL; -} - -/* Python/fileutils.c */ -wchar_t *Py_DecodeLocale(const char* arg, size_t *size) -{ - wchar_t *w; - __coverity_tainted_data_sink__(arg); - __coverity_tainted_data_sink__(size); - return w; -} - -/* Python/marshal.c */ - -static Py_ssize_t r_string(char *s, Py_ssize_t n, RFILE *p) -{ - __coverity_tainted_string_argument__(s); - return 0; -} - -static long r_long(RFILE *p) -{ - long l; - unsigned char buffer[4]; - - r_string((char *)buffer, 4, p); - __coverity_tainted_string_sanitize_content__(buffer); - l = (long)buffer; - return l; -} - -/* Coverity doesn't understand that fdopendir() may take ownership of fd. */ - -DIR *fdopendir(int fd) -{ - DIR *d; - if (d) { - __coverity_close__(fd); - } - return d; -} - -/* Modules/_datetime.c - * - * Coverity thinks that the input values for these function come from a - * tainted source PyDateTime_DATE_GET_* macros use bit shifting. - */ -static PyObject * -build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) -{ - PyObject *result; - - __coverity_tainted_data_sanitize__(y); - __coverity_tainted_data_sanitize__(m); - __coverity_tainted_data_sanitize__(d); - __coverity_tainted_data_sanitize__(hh); - __coverity_tainted_data_sanitize__(mm); - __coverity_tainted_data_sanitize__(ss); - __coverity_tainted_data_sanitize__(dstflag); - - return result; -} - -static int -ymd_to_ord(int year, int month, int day) -{ - int ord = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return ord; -} - -static int -normalize_date(int *year, int *month, int *day) -{ - __coverity_tainted_data_sanitize__(*year); - __coverity_tainted_data_sanitize__(*month); - __coverity_tainted_data_sanitize__(*day); - - return 0; -} - -static int -weekday(int year, int month, int day) -{ - int w = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return w; -} - diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index f7aea9e8f990ba..d54b1fbe251378 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -112,42 +112,42 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "ad7623a44e1b6e42df47ba8f16b2b0435ac605650b5054077c4355a30473074c" + "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.14.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.14.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tcl-core", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.14.0" + "versionInfo": "8.6.15.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e8d5cbe97952037962518b69aba85e324d80aa189054c163ab0ee764a448e802" + "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.14.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.14.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.14.0" + "versionInfo": "8.6.15.0" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index fe0a5e44f8fb15..62978261745d79 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2536,3 +2536,5 @@ added = '3.14' [const.Py_TP_USE_SPEC] added = '3.14' +[function.PyUnicode_Equal] + added = '3.14' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 9aa398a80efa1b..52c0f883d383db 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 870084100a1b85..0a769c46b87ac8 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1387,7 +1387,7 @@ FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) default: assert (0); } - assert(_Py_IsImmortalLoose(ret)); + assert(_Py_IsImmortal(ret)); return ret; } diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 32373f0799bfeb..471b42badc8e8c 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -979,6 +979,30 @@ _codecs_register_error_impl(PyObject *module, const char *errors, Py_RETURN_NONE; } +/*[clinic input] +_codecs._unregister_error -> bool + errors: str + / + +Un-register the specified error handler for the error handling `errors'. + +Only custom error handlers can be un-registered. An exception is raised +if the error handling is a built-in one (e.g., 'strict'), or if an error +occurs. + +Otherwise, this returns True if a custom handler has been successfully +un-registered, and False if no custom handler for the specified error +handling exists. + +[clinic start generated code]*/ + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors) +/*[clinic end generated code: output=28c22be667465503 input=a63ab9e9ce1686d4]*/ +{ + return _PyCodec_UnregisterError(errors); +} + /*[clinic input] _codecs.lookup_error name: str @@ -1044,6 +1068,7 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_ENCODE_METHODDEF _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF + _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index fbfed59995c21e..aef04248c7e73c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2179,6 +2179,8 @@ typedef struct { PyObject *default_factory; } defdictobject; +static PyType_Spec defdict_spec; + PyDoc_STRVAR(defdict_missing_doc, "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\ if self.default_factory is None: raise KeyError((key,))\n\ @@ -2358,23 +2360,16 @@ defdict_or(PyObject* left, PyObject* right) { PyObject *self, *other; - // Find module state - PyTypeObject *tp = Py_TYPE(left); - PyObject *mod = PyType_GetModuleByDef(tp, &_collectionsmodule); - if (mod == NULL) { - PyErr_Clear(); - tp = Py_TYPE(right); - mod = PyType_GetModuleByDef(tp, &_collectionsmodule); + int ret = PyType_GetBaseByToken(Py_TYPE(left), &defdict_spec, NULL); + if (ret < 0) { + return NULL; } - assert(mod != NULL); - collections_state *state = get_module_state(mod); - - if (PyObject_TypeCheck(left, state->defdict_type)) { + if (ret) { self = left; other = right; } else { - assert(PyObject_TypeCheck(right, state->defdict_type)); + assert(PyType_GetBaseByToken(Py_TYPE(right), &defdict_spec, NULL) == 1); self = right; other = left; } @@ -2454,6 +2449,7 @@ passed to the dict constructor, including keyword arguments.\n\ #define DEFERRED_ADDRESS(ADDR) 0 static PyType_Slot defdict_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, defdict_dealloc}, {Py_tp_repr, defdict_repr}, {Py_nb_or, defdict_or}, diff --git a/Modules/_csv.c b/Modules/_csv.c index a623ea449da779..1a4dc3f1f55ace 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -14,6 +14,7 @@ module instead. #endif #include "Python.h" +#include "pycore_pyatomic_ft_wrappers.h" #include // offsetof() #include @@ -34,7 +35,7 @@ typedef struct { PyTypeObject *dialect_type; PyTypeObject *reader_type; PyTypeObject *writer_type; - long field_limit; /* max parsed field size */ + Py_ssize_t field_limit; /* max parsed field size */ PyObject *str_write; } _csvstate; @@ -367,6 +368,8 @@ static struct PyMemberDef Dialect_memberlist[] = { { NULL } }; +#undef D_OFF + static PyGetSetDef Dialect_getsetlist[] = { { "delimiter", (getter)Dialect_get_delimiter}, { "escapechar", (getter)Dialect_get_escapechar}, @@ -502,6 +505,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIALECT_GETATTR(skipinitialspace, "skipinitialspace"); DIALECT_GETATTR(strict, "strict"); } +#undef DIALECT_GETATTR /* check types and convert to C values */ #define DIASET(meth, name, target, src, dflt) \ @@ -515,6 +519,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL); DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false); DIASET(_set_bool, "strict", &self->strict, strict, false); +#undef DIASET /* validate options */ if (dialect_check_quoting(self->quoting)) @@ -702,10 +707,11 @@ parse_grow_buff(ReaderObj *self) static int parse_add_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) { - if (self->field_len >= module_state->field_limit) { + Py_ssize_t field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); + if (self->field_len >= field_limit) { PyErr_Format(module_state->error_obj, - "field larger than field limit (%ld)", - module_state->field_limit); + "field larger than field limit (%zd)", + field_limit); return -1; } if (self->field_len == self->field_size && !parse_grow_buff(self)) @@ -1026,6 +1032,8 @@ static struct PyMemberDef Reader_memberlist[] = { { NULL } }; +#undef R_OFF + static PyType_Slot Reader_Type_slots[] = { {Py_tp_doc, (char*)Reader_Type_doc}, @@ -1441,6 +1449,8 @@ static struct PyMemberDef Writer_memberlist[] = { { NULL } }; +#undef W_OFF + static int Writer_traverse(WriterObj *self, visitproc visit, void *arg) { @@ -1651,20 +1661,20 @@ _csv_field_size_limit_impl(PyObject *module, PyObject *new_limit) /*[clinic end generated code: output=f2799ecd908e250b input=cec70e9226406435]*/ { _csvstate *module_state = get_csv_state(module); - long old_limit = module_state->field_limit; + Py_ssize_t old_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); if (new_limit != NULL) { if (!PyLong_CheckExact(new_limit)) { PyErr_Format(PyExc_TypeError, "limit must be an integer"); return NULL; } - module_state->field_limit = PyLong_AsLong(new_limit); - if (module_state->field_limit == -1 && PyErr_Occurred()) { - module_state->field_limit = old_limit; + Py_ssize_t new_limit_value = PyLong_AsSsize_t(new_limit); + if (new_limit_value == -1 && PyErr_Occurred()) { return NULL; } + FT_ATOMIC_STORE_SSIZE_RELAXED(module_state->field_limit, new_limit_value); } - return PyLong_FromLong(old_limit); + return PyLong_FromSsize_t(old_limit); } static PyType_Slot error_slots[] = { diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 951e6914ba67a4..8435ee4090b9e5 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4732,7 +4732,7 @@ Array_subscript(PyObject *myself, PyObject *item) char *dest; if (slicelen <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { return PyBytes_FromStringAndSize(ptr + start, slicelen); @@ -4756,7 +4756,7 @@ Array_subscript(PyObject *myself, PyObject *item) wchar_t *dest; if (slicelen <= 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { return PyUnicode_FromWideChar(ptr + start, slicelen); @@ -5418,7 +5418,7 @@ Pointer_subscript(PyObject *myself, PyObject *item) char *dest; if (len <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { return PyBytes_FromStringAndSize(ptr + start, len); @@ -5438,7 +5438,7 @@ Pointer_subscript(PyObject *myself, PyObject *item) wchar_t *dest; if (len <= 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { return PyUnicode_FromWideChar(ptr + start, len); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index c9ee5687c2b5d9..27d5df08de933e 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -105,8 +105,9 @@ static const char PyCursesVersion[] = "2.2"; #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_structseq.h" // _PyStructSequence_NewType() +#include "pycore_capsule.h" // _PyCapsule_SetTraverse() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_structseq.h" // _PyStructSequence_NewType() #ifdef __hpux #define STRICT_SYSV_CURSES @@ -159,15 +160,41 @@ typedef chtype attr_t; /* No attr_t type is available */ #define _CURSES_PAIR_CONTENT_FUNC pair_content #endif /* _NCURSES_EXTENDED_COLOR_FUNCS */ +typedef struct { + PyObject *error; // curses exception type + PyTypeObject *window_type; // exposed by PyCursesWindow_Type +} cursesmodule_state; + +static inline cursesmodule_state * +get_cursesmodule_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (cursesmodule_state *)state; +} + +static inline cursesmodule_state * +get_cursesmodule_state_by_cls(PyTypeObject *cls) +{ + void *state = PyType_GetModuleState(cls); + assert(state != NULL); + return (cursesmodule_state *)state; +} + +static inline cursesmodule_state * +get_cursesmodule_state_by_win(PyCursesWindowObject *win) +{ + return get_cursesmodule_state_by_cls(Py_TYPE(win)); +} + /*[clinic input] module _curses -class _curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type" +class _curses.window "PyCursesWindowObject *" "clinic_state()->window_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=43265c372c2887d6]*/ - -/* Definition of exception curses.error */ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ae6cb623018f2cbc]*/ -static PyObject *PyCursesError; +/* Indicate whether the module has already been loaded or not. */ +static int curses_module_loaded = 0; /* Tells whether setupterm() has been called to initialise terminfo. */ static int curses_setupterm_called = FALSE; @@ -180,53 +207,118 @@ static int curses_start_color_called = FALSE; static const char *curses_screen_encoding = NULL; -/* Utility Macros */ -#define PyCursesSetupTermCalled \ +/* Utility Checking Procedures */ + +/* + * Function to check that 'funcname' has been called by testing + * the 'called' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise, this returns 1. + * + * Since this function can be called in functions that do not + * have a direct access to the module's state, '_curses.error' + * is imported on demand. + */ +static inline int +_PyCursesCheckFunction(int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + PyObject *exc = _PyImport_GetModuleAttrString("_curses", "error"); + if (exc != NULL) { + PyErr_Format(exc, "must call %s() first", funcname); + Py_DECREF(exc); + } + assert(PyErr_Occurred()); + return 0; +} + +/* + * Function to check that 'funcname' has been called by testing + * the 'called'' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise this returns 1. + * + * The exception type is obtained from the 'module' state. + */ +static inline int +_PyCursesStatefulCheckFunction(PyObject *module, int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "must call %s() first", funcname); + return 0; +} + +#define PyCursesStatefulSetupTermCalled(MODULE) \ do { \ - if (curses_setupterm_called != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call (at least) setupterm() first"); \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_setupterm_called, \ + "setupterm")) \ + { \ return 0; \ } \ } while (0) -#define PyCursesInitialised \ - do { \ - if (curses_initscr_called != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call initscr() first"); \ - return 0; \ - } \ +#define PyCursesStatefulInitialised(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_initscr_called, \ + "initscr")) \ + { \ + return 0; \ + } \ } while (0) -#define PyCursesInitialisedColor \ - do { \ - if (curses_start_color_called != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call start_color() first"); \ - return 0; \ - } \ +#define PyCursesStatefulInitialisedColor(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_start_color_called, \ + "start_color")) \ + { \ + return 0; \ + } \ } while (0) /* Utility Functions */ +static inline void +_PyCursesSetError(cursesmodule_state *state, const char *funcname) +{ + if (funcname == NULL) { + PyErr_SetString(state->error, catchall_ERR); + } + else { + PyErr_Format(state->error, "%s() returned ERR", funcname); + } +} + /* * Check the return code from a curses function and return None - * or raise an exception as appropriate. These are exported using the - * capsule API. + * or raise an exception as appropriate. */ static PyObject * -PyCursesCheckERR(int code, const char *fname) +PyCursesCheckERR(PyObject *module, int code, const char *fname) { if (code != ERR) { Py_RETURN_NONE; } else { - if (fname == NULL) { - PyErr_SetString(PyCursesError, catchall_ERR); - } else { - PyErr_Format(PyCursesError, "%s() returned ERR", fname); - } + cursesmodule_state *state = get_cursesmodule_state(module); + _PyCursesSetError(state, fname); + return NULL; + } +} + +static PyObject * +PyCursesCheckERR_ForWin(PyCursesWindowObject *win, int code, const char *fname) +{ + if (code != ERR) { + Py_RETURN_NONE; + } else { + cursesmodule_state *state = get_cursesmodule_state_by_win(win); + _PyCursesSetError(state, fname); return NULL; } } @@ -550,35 +642,10 @@ class component_converter(CConverter): [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=38e9be01d33927fb]*/ -/* Function versions of the 3 functions for testing whether curses has been - initialised or not. */ - -static int func_PyCursesSetupTermCalled(void) -{ - PyCursesSetupTermCalled; - return 1; -} - -static int func_PyCursesInitialised(void) -{ - PyCursesInitialised; - return 1; -} - -static int func_PyCursesInitialisedColor(void) -{ - PyCursesInitialisedColor; - return 1; -} - /***************************************************************************** The Window Object ******************************************************************************/ -/* Definition of the window type */ - -PyTypeObject PyCursesWindow_Type; - /* Function prototype macros for Window object X - function name @@ -590,7 +657,7 @@ PyTypeObject PyCursesWindow_Type; #define Window_NoArgNoReturnFunction(X) \ static PyObject *PyCursesWindow_ ## X \ (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { return PyCursesCheckERR(X(self->win), # X); } + { return PyCursesCheckERR_ForWin(self, X(self->win), # X); } #define Window_NoArgTrueFalseFunction(X) \ static PyObject * PyCursesWindow_ ## X \ @@ -625,7 +692,7 @@ PyTypeObject PyCursesWindow_Type; { \ TYPE arg1; \ if (!PyArg_ParseTuple(args,PARSESTR, &arg1)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1), # X); } + return PyCursesCheckERR_ForWin(self, X(self->win, arg1), # X); } #define Window_TwoArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ @@ -633,7 +700,7 @@ PyTypeObject PyCursesWindow_Type; { \ TYPE arg1, arg2; \ if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1, arg2), # X); } + return PyCursesCheckERR_ForWin(self, X(self->win, arg1, arg2), # X); } /* ------------- WINDOW routines --------------- */ @@ -688,10 +755,9 @@ Window_TwoArgNoReturnFunction(wresize, int, "ii;lines,columns") /* Allocation and deallocation of Window Objects */ static PyObject * -PyCursesWindow_New(WINDOW *win, const char *encoding) +PyCursesWindow_New(cursesmodule_state *state, + WINDOW *win, const char *encoding) { - PyCursesWindowObject *wo; - if (encoding == NULL) { #if defined(MS_WINDOWS) char *buffer[100]; @@ -703,15 +769,20 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) } #elif defined(CODESET) const char *codeset = nl_langinfo(CODESET); - if (codeset != NULL && codeset[0] != 0) + if (codeset != NULL && codeset[0] != 0) { encoding = codeset; + } #endif - if (encoding == NULL) + if (encoding == NULL) { encoding = "utf-8"; + } } - wo = PyObject_New(PyCursesWindowObject, &PyCursesWindow_Type); - if (wo == NULL) return NULL; + PyCursesWindowObject *wo = PyObject_GC_New(PyCursesWindowObject, + state->window_type); + if (wo == NULL) { + return NULL; + } wo->win = win; wo->encoding = _PyMem_Strdup(encoding); if (wo->encoding == NULL) { @@ -719,12 +790,16 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) PyErr_NoMemory(); return NULL; } + PyObject_GC_Track((PyObject *)wo); return (PyObject *)wo; } static void -PyCursesWindow_Dealloc(PyCursesWindowObject *wo) +PyCursesWindow_dealloc(PyObject *self) { + PyTypeObject *window_type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + PyCursesWindowObject *wo = (PyCursesWindowObject *)self; if (wo->win != stdscr && wo->win != NULL) { // silently ignore errors in delwin(3) (void)delwin(wo->win); @@ -732,7 +807,15 @@ PyCursesWindow_Dealloc(PyCursesWindowObject *wo) if (wo->encoding != NULL) { PyMem_Free(wo->encoding); } - PyObject_Free(wo); + window_type->tp_free(self); + Py_DECREF(window_type); +} + +static int +PyCursesWindow_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; } /* Addch, Addstr, Addnstr */ @@ -807,7 +890,7 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, else { return NULL; } - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -887,7 +970,7 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -970,7 +1053,7 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -994,7 +1077,7 @@ _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr) if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; - return PyCursesCheckERR(wbkgd(self->win, bkgd | attr), "bkgd"); + return PyCursesCheckERR_ForWin(self, wbkgd(self->win, bkgd | attr), "bkgd"); } /*[clinic input] @@ -1010,7 +1093,7 @@ static PyObject * _curses_window_attroff_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=8a2fcd4df682fc64 input=786beedf06a7befe]*/ { - return PyCursesCheckERR(wattroff(self->win, (attr_t)attr), "attroff"); + return PyCursesCheckERR_ForWin(self, wattroff(self->win, (attr_t)attr), "attroff"); } /*[clinic input] @@ -1026,7 +1109,7 @@ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=7afea43b237fa870 input=5a88fba7b1524f32]*/ { - return PyCursesCheckERR(wattron(self->win, (attr_t)attr), "attron"); + return PyCursesCheckERR_ForWin(self, wattron(self->win, (attr_t)attr), "attron"); } /*[clinic input] @@ -1042,7 +1125,7 @@ static PyObject * _curses_window_attrset_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=84e379bff20c0433 input=42e400c0d0154ab5]*/ { - return PyCursesCheckERR(wattrset(self->win, (attr_t)attr), "attrset"); + return PyCursesCheckERR_ForWin(self, wattrset(self->win, (attr_t)attr), "attrset"); } /*[clinic input] @@ -1068,7 +1151,7 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, return NULL; wbkgdset(self->win, bkgd | attr); - return PyCursesCheckERR(0, "bkgdset"); + return PyCursesCheckERR_ForWin(self, 0, "bkgdset"); } /*[clinic input] @@ -1268,7 +1351,7 @@ PyCursesWindow_ChgAt(PyCursesWindowObject *self, PyObject *args) rtn = wchgat(self->win,num,attr,color,NULL); touchline(self->win,y,1); } - return PyCursesCheckERR(rtn, "chgat"); + return PyCursesCheckERR_ForWin(self, rtn, "chgat"); } #endif @@ -1292,10 +1375,10 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ { if (!group_right_1) { - return PyCursesCheckERR(wdelch(self->win), "wdelch"); + return PyCursesCheckERR_ForWin(self, wdelch(self->win), "wdelch"); } else { - return PyCursesCheckERR(py_mvwdelch(self->win, y, x), "mvwdelch"); + return PyCursesCheckERR_ForWin(self, py_mvwdelch(self->win, y, x), "mvwdelch"); } } @@ -1331,11 +1414,13 @@ _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, win = derwin(self->win,nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -1363,13 +1448,15 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, #ifdef py_is_pad if (py_is_pad(self->win)) { - return PyCursesCheckERR(pechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + pechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } else #endif - return PyCursesCheckERR(wechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + wechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } #ifdef NCURSES_MOUSE_VERSION @@ -1480,8 +1567,10 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, if (rtn == ERR) { /* getch() returns ERR in nodelay mode */ PyErr_CheckSignals(); - if (!PyErr_Occurred()) - PyErr_SetString(PyCursesError, "no input"); + if (!PyErr_Occurred()) { + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); + } return NULL; } else if (rtn <= 255) { #ifdef NCURSES_VERSION_MAJOR @@ -1539,7 +1628,8 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, return NULL; /* get_wch() returns ERR in nodelay mode */ - PyErr_SetString(PyCursesError, "no input"); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); return NULL; } if (ct == KEY_CODE_YES) @@ -1663,10 +1753,10 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) { - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } } - return PyCursesCheckERR(whline(self->win, ch_ | (attr_t)attr, n), "hline"); + return PyCursesCheckERR_ForWin(self, whline(self->win, ch_ | (attr_t)attr, n), "hline"); } /*[clinic input] @@ -1713,7 +1803,7 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, rtn = mvwinsch(self->win, y, x, ch_ | (attr_t)attr); } - return PyCursesCheckERR(rtn, "insch"); + return PyCursesCheckERR_ForWin(self, rtn, "insch"); } /*[clinic input] @@ -1890,7 +1980,7 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -1975,7 +2065,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -2052,7 +2142,8 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "noutrefresh() called for a pad " "requires 6 arguments"); return NULL; @@ -2061,7 +2152,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) rtn = pnoutrefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "pnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "pnoutrefresh"); } if (group_right_1) { PyErr_SetString(PyExc_TypeError, @@ -2072,13 +2163,13 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) Py_BEGIN_ALLOW_THREADS rtn = wnoutrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "wnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "wnoutrefresh"); } /*[clinic input] _curses.window.overlay - destwin: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") + destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") [ sminrow: int @@ -2107,25 +2198,25 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=7edd23ad22cc1984]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=6e4b32a7c627a356]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, TRUE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overlay(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overlay"); + return PyCursesCheckERR_ForWin(self, rtn, "overlay"); } } /*[clinic input] _curses.window.overwrite - destwin: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") + destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") [ sminrow: int @@ -2155,18 +2246,18 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=ea5de1b35cd948e0]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=d83dd8b24ff2bcc9]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, FALSE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overwrite(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overwrite"); + return PyCursesCheckERR_ForWin(self, rtn, "overwrite"); } } @@ -2195,7 +2286,7 @@ _curses_window_putwin(PyCursesWindowObject *self, PyObject *file) return PyErr_SetFromErrno(PyExc_OSError); if (_Py_set_inheritable(fileno(fp), 0, NULL) < 0) goto exit; - res = PyCursesCheckERR(putwin(self->win, fp), "putwin"); + res = PyCursesCheckERR_ForWin(self, putwin(self->win, fp), "putwin"); if (res == NULL) goto exit; fseek(fp, 0, 0); @@ -2234,7 +2325,7 @@ static PyObject * _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) /*[clinic end generated code: output=ea216e334f9ce1b4 input=152155e258a77a7a]*/ { - return PyCursesCheckERR(wredrawln(self->win,beg,num), "redrawln"); + return PyCursesCheckERR_ForWin(self, wredrawln(self->win,beg,num), "redrawln"); } /*[clinic input] @@ -2276,7 +2367,8 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "refresh() for a pad requires 6 arguments"); return NULL; } @@ -2284,7 +2376,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, rtn = prefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } #endif if (group_right_1) { @@ -2295,7 +2387,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, Py_BEGIN_ALLOW_THREADS rtn = wrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } /*[clinic input] @@ -2317,7 +2409,7 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, int bottom) /*[clinic end generated code: output=486ab5db218d2b1a input=1b517b986838bf0e]*/ { - return PyCursesCheckERR(wsetscrreg(self->win, top, bottom), "wsetscrreg"); + return PyCursesCheckERR_ForWin(self, wsetscrreg(self->win, top, bottom), "wsetscrreg"); } /*[clinic input] @@ -2358,11 +2450,13 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, win = subwin(self->win, nlines, ncols, begin_y, begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, self->encoding); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + return PyCursesWindow_New(state, win, self->encoding); } /*[clinic input] @@ -2385,10 +2479,10 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ { if (!group_right_1) { - return PyCursesCheckERR(scroll(self->win), "scroll"); + return PyCursesCheckERR_ForWin(self, scroll(self->win), "scroll"); } else { - return PyCursesCheckERR(wscrl(self->win, lines), "scroll"); + return PyCursesCheckERR_ForWin(self, wscrl(self->win, lines), "scroll"); } } @@ -2414,10 +2508,10 @@ _curses_window_touchline_impl(PyCursesWindowObject *self, int start, /*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ { if (!group_right_1) { - return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); + return PyCursesCheckERR_ForWin(self, touchline(self->win, start, count), "touchline"); } else { - return PyCursesCheckERR(wtouchln(self->win, start, count, changed), "touchline"); + return PyCursesCheckERR_ForWin(self, wtouchln(self->win, start, count, changed), "touchline"); } } @@ -2457,9 +2551,9 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } - return PyCursesCheckERR(wvline(self->win, ch_ | (attr_t)attr, n), "vline"); + return PyCursesCheckERR_ForWin(self, wvline(self->win, ch_ | (attr_t)attr, n), "vline"); } static PyObject * @@ -2500,9 +2594,11 @@ PyCursesWindow_set_encoding(PyCursesWindowObject *self, PyObject *value, void *P return 0; } +#define clinic_state() (get_cursesmodule_state_by_cls(Py_TYPE(self))) #include "clinic/_cursesmodule.c.h" +#undef clinic_state -static PyMethodDef PyCursesWindow_Methods[] = { +static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ADDCH_METHODDEF _CURSES_WINDOW_ADDNSTR_METHODDEF _CURSES_WINDOW_ADDSTR_METHODDEF @@ -2596,43 +2692,28 @@ static PyGetSetDef PyCursesWindow_getsets[] = { {NULL, NULL, NULL, NULL } /* sentinel */ }; -/* -------------------------------------------------------*/ +static PyType_Slot PyCursesWindow_Type_slots[] = { + {Py_tp_methods, PyCursesWindow_methods}, + {Py_tp_getset, PyCursesWindow_getsets}, + {Py_tp_dealloc, PyCursesWindow_dealloc}, + {Py_tp_traverse, PyCursesWindow_traverse}, + {0, NULL} +}; -PyTypeObject PyCursesWindow_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_curses.window", /*tp_name*/ - sizeof(PyCursesWindowObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCursesWindow_Dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - (getattrfunc)0, /*tp_getattr*/ - (setattrfunc)0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - PyCursesWindow_Methods, /*tp_methods*/ - 0, /* tp_members */ - PyCursesWindow_getsets, /* tp_getset */ +static PyType_Spec PyCursesWindow_Type_spec = { + .name = "_curses.window", + .basicsize = sizeof(PyCursesWindowObject), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HEAPTYPE + | Py_TPFLAGS_HAVE_GC, + .slots = PyCursesWindow_Type_slots }; -/* Function Prototype Macros - They are ugly but very, very useful. ;-) +/* -------------------------------------------------------*/ + +/* Function Body Macros - They are ugly but very, very useful. ;-) X - function name TYPE - parameter Type @@ -2642,37 +2723,37 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgNoReturnFunctionBody(X) \ { \ - PyCursesInitialised; \ - return PyCursesCheckERR(X(), # X); } + PyCursesStatefulInitialised(module); \ + return PyCursesCheckERR(module, X(), # X); } #define NoArgOrFlagNoReturnFunctionBody(X, flag) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ if (flag) \ - return PyCursesCheckERR(X(), # X); \ + return PyCursesCheckERR(module, X(), # X); \ else \ - return PyCursesCheckERR(no ## X(), # X); \ + return PyCursesCheckERR(module, no ## X(), # X); \ } #define NoArgReturnIntFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ return PyLong_FromLong((long) X()); } #define NoArgReturnStringFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ return PyBytes_FromString(X()); } #define NoArgTrueFalseFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ return PyBool_FromLong(X()); } #define NoArgNoReturnVoidFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ X(); \ Py_RETURN_NONE; } @@ -2770,12 +2851,13 @@ _curses_color_content_impl(PyObject *module, int color_number) { _CURSES_COLOR_VAL_TYPE r,g,b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_COLOR_CONTENT_FUNC(color_number, &r, &g, &b) == ERR) { - PyErr_Format(PyCursesError, "%s() returned ERR", - Py_STRINGIFY(_COLOR_CONTENT_FUNC)); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", + Py_STRINGIFY(_COLOR_CONTENT_FUNC)); return NULL; } @@ -2799,8 +2881,8 @@ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) /*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(COLOR_PAIR(pair_number)); } @@ -2826,10 +2908,10 @@ _curses_curs_set_impl(PyObject *module, int visibility) { int erg; - PyCursesInitialised; + PyCursesStatefulInitialised(module); erg = curs_set(visibility); - if (erg == ERR) return PyCursesCheckERR(erg, "curs_set"); + if (erg == ERR) return PyCursesCheckERR(module, erg, "curs_set"); return PyLong_FromLong((long) erg); } @@ -2878,9 +2960,9 @@ static PyObject * _curses_delay_output_impl(PyObject *module, int ms) /*[clinic end generated code: output=b6613a67f17fa4f4 input=5316457f5f59196c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(delay_output(ms), "delay_output"); + return PyCursesCheckERR(module, delay_output(ms), "delay_output"); } /*[clinic input] @@ -2934,7 +3016,7 @@ _curses_erasechar_impl(PyObject *module) { char ch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); ch = erasechar(); @@ -2984,7 +3066,7 @@ _curses_getsyx_impl(PyObject *module) int x = 0; int y = 0; - PyCursesInitialised; + PyCursesStatefulInitialised(module); getsyx(y, x); @@ -3009,11 +3091,12 @@ _curses_getmouse_impl(PyObject *module) int rtn; MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); rtn = getmouse( &event ); if (rtn == ERR) { - PyErr_SetString(PyCursesError, "getmouse() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "getmouse() returned ERR"); return NULL; } return Py_BuildValue("(hiiik)", @@ -3044,14 +3127,14 @@ _curses_ungetmouse_impl(PyObject *module, short id, int x, int y, int z, { MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); event.id = id; event.x = x; event.y = y; event.z = z; event.bstate = bstate; - return PyCursesCheckERR(ungetmouse(&event), "ungetmouse"); + return PyCursesCheckERR(module, ungetmouse(&event), "ungetmouse"); } #endif @@ -3077,7 +3160,7 @@ _curses_getwin(PyObject *module, PyObject *file) WINDOW *win; PyObject *res = NULL; - PyCursesInitialised; + PyCursesStatefulInitialised(module); fp = tmpfile(); if (fp == NULL) @@ -3107,10 +3190,12 @@ _curses_getwin(PyObject *module, PyObject *file) fseek(fp, 0, 0); win = getwin(fp); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); goto error; } - res = PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + res = PyCursesWindow_New(state, win, NULL); error: fclose(fp); @@ -3133,9 +3218,9 @@ static PyObject * _curses_halfdelay_impl(PyObject *module, unsigned char tenths) /*[clinic end generated code: output=e92cdf0ef33c0663 input=e42dce7259c15100]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(halfdelay(tenths), "halfdelay"); + return PyCursesCheckERR(module, halfdelay(tenths), "halfdelay"); } /*[clinic input] @@ -3186,7 +3271,7 @@ static PyObject * _curses_has_key_impl(PyObject *module, int key) /*[clinic end generated code: output=19ad48319414d0b1 input=78bd44acf1a4997c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(has_key(key)); } @@ -3217,10 +3302,11 @@ _curses_init_color_impl(PyObject *module, int color_number, short r, short g, short b) /*[clinic end generated code: output=d7ed71b2d818cdf2 input=ae2b8bea0f152c80]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); - return PyCursesCheckERR(_CURSES_INIT_COLOR_FUNC(color_number, r, g, b), + return PyCursesCheckERR(module, + _CURSES_INIT_COLOR_FUNC(color_number, r, g, b), Py_STRINGIFY(_CURSES_INIT_COLOR_FUNC)); } @@ -3245,8 +3331,8 @@ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) /*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_INIT_PAIR_FUNC(pair_number, fg, bg) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3255,7 +3341,8 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_INIT_PAIR_FUNC)); } return NULL; @@ -3280,13 +3367,15 @@ _curses_initscr_impl(PyObject *module) if (curses_initscr_called) { wrefresh(stdscr); - return (PyObject *)PyCursesWindow_New(stdscr, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, stdscr, NULL); } win = initscr(); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } @@ -3382,12 +3471,13 @@ _curses_initscr_impl(PyObject *module) SetDictInt("COLS", COLS); #undef SetDictInt - PyCursesWindowObject *winobj = (PyCursesWindowObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyObject *winobj = PyCursesWindow_New(state, win, NULL); if (winobj == NULL) { return NULL; } - curses_screen_encoding = winobj->encoding; - return (PyObject *)winobj; + curses_screen_encoding = ((PyCursesWindowObject *)winobj)->encoding; + return winobj; } /*[clinic input] @@ -3415,9 +3505,8 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) sys_stdout = PySys_GetObject("stdout"); if (sys_stdout == NULL || sys_stdout == Py_None) { - PyErr_SetString( - PyCursesError, - "lost sys.stdout"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "lost sys.stdout"); return NULL; } @@ -3437,7 +3526,8 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) s = "setupterm: could not find terminfo database"; } - PyErr_SetString(PyCursesError,s); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, s); return NULL; } @@ -3487,7 +3577,7 @@ _curses_set_escdelay_impl(PyObject *module, int ms) return NULL; } - return PyCursesCheckERR(set_escdelay(ms), "set_escdelay"); + return PyCursesCheckERR(module, set_escdelay(ms), "set_escdelay"); } /*[clinic input] @@ -3526,7 +3616,7 @@ _curses_set_tabsize_impl(PyObject *module, int size) return NULL; } - return PyCursesCheckERR(set_tabsize(size), "set_tabsize"); + return PyCursesCheckERR(module, set_tabsize(size), "set_tabsize"); } #endif @@ -3542,9 +3632,9 @@ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(intrflush(NULL, flag), "intrflush"); + return PyCursesCheckERR(module, intrflush(NULL, flag), "intrflush"); } /*[clinic input] @@ -3575,7 +3665,7 @@ static PyObject * _curses_is_term_resized_impl(PyObject *module, int nlines, int ncols) /*[clinic end generated code: output=aafe04afe50f1288 input=ca9c0bd0fb8ab444]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(is_term_resized(nlines, ncols)); } @@ -3597,7 +3687,7 @@ _curses_keyname_impl(PyObject *module, int key) { const char *knp; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (key < 0) { PyErr_SetString(PyExc_ValueError, "invalid key number"); @@ -3655,9 +3745,9 @@ static PyObject * _curses_meta_impl(PyObject *module, int yes) /*[clinic end generated code: output=22f5abda46a605d8 input=cfe7da79f51d0e30]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(meta(stdscr, yes), "meta"); + return PyCursesCheckERR(module, meta(stdscr, yes), "meta"); } #ifdef NCURSES_MOUSE_VERSION @@ -3679,9 +3769,9 @@ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) /*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(mouseinterval(interval), "mouseinterval"); + return PyCursesCheckERR(module, mouseinterval(interval), "mouseinterval"); } /*[clinic input] @@ -3704,7 +3794,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) { mmask_t oldmask, availmask; - PyCursesInitialised; + PyCursesStatefulInitialised(module); availmask = mousemask((mmask_t)newmask, &oldmask); return Py_BuildValue("(kk)", (unsigned long)availmask, (unsigned long)oldmask); @@ -3725,7 +3815,7 @@ static int _curses_napms_impl(PyObject *module, int ms) /*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return napms(ms); } @@ -3749,16 +3839,18 @@ _curses_newpad_impl(PyObject *module, int nlines, int ncols) { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newpad(nlines, ncols); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -3789,15 +3881,17 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newwin(nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -3901,8 +3995,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) { _CURSES_COLOR_NUM_TYPE f, b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_PAIR_CONTENT_FUNC(pair_number, &f, &b) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3911,7 +4005,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_PAIR_CONTENT_FUNC)); } return NULL; @@ -3935,8 +4030,8 @@ static PyObject * _curses_pair_number_impl(PyObject *module, int attr) /*[clinic end generated code: output=85bce7d65c0aa3f4 input=d478548e33f5e61a]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(PAIR_NUMBER(attr)); } @@ -3956,7 +4051,7 @@ static PyObject * _curses_putp_impl(PyObject *module, const char *string) /*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ { - return PyCursesCheckERR(putp(string), "putp"); + return PyCursesCheckERR(module, putp(string), "putp"); } /*[clinic input] @@ -3976,7 +4071,7 @@ static PyObject * _curses_qiflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=9167e862f760ea30 input=6ec8b3e2b717ec40]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (flag) { qiflush(); @@ -4036,7 +4131,7 @@ update_lines_cols(PyObject *private_module) error: Py_XDECREF(o); - Py_DECREF(exposed_module); + Py_XDECREF(exposed_module); return 0; } @@ -4113,9 +4208,9 @@ NoArgNoReturnFunctionBody(resetty) /*[clinic input] _curses.resizeterm - nlines: int + nlines: short Height. - ncols: int + ncols: short Width. / @@ -4126,14 +4221,14 @@ window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * -_curses_resizeterm_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=56d6bcc5194ad055 input=0fca02ebad5ffa82]*/ +_curses_resizeterm_impl(PyObject *module, short nlines, short ncols) +/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resizeterm(nlines, ncols), "resizeterm"); + result = PyCursesCheckERR(module, resizeterm(nlines, ncols), "resizeterm"); if (!result) return NULL; if (!update_lines_cols(module)) { @@ -4149,9 +4244,9 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols) /*[clinic input] _curses.resize_term - nlines: int + nlines: short Height. - ncols: int + ncols: short Width. / @@ -4165,14 +4260,14 @@ without additional interaction with the application. [clinic start generated code]*/ static PyObject * -_curses_resize_term_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=9e26d8b9ea311ed2 input=2197edd05b049ed4]*/ +_curses_resize_term_impl(PyObject *module, short nlines, short ncols) +/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/ { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resize_term(nlines, ncols), "resize_term"); + result = PyCursesCheckERR(module, resize_term(nlines, ncols), "resize_term"); if (!result) return NULL; if (!update_lines_cols(module)) { @@ -4213,7 +4308,7 @@ static PyObject * _curses_setsyx_impl(PyObject *module, int y, int x) /*[clinic end generated code: output=23dcf753511a2464 input=fa7f2b208e10a557]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); setsyx(y,x); @@ -4238,10 +4333,11 @@ static PyObject * _curses_start_color_impl(PyObject *module) /*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (start_color() == ERR) { - PyErr_SetString(PyCursesError, "start_color() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "start_color() returned ERR"); return NULL; } @@ -4310,7 +4406,7 @@ static PyObject * _curses_tigetflag_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=8853c0e55542195b input=b0787af9e3e9a6ce]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetflag( (char *)capname ) ); } @@ -4332,7 +4428,7 @@ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetnum( (char *)capname ) ); } @@ -4354,7 +4450,7 @@ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); capname = tigetstr( (char *)capname ); if (capname == NULL || capname == (char*) -1) { @@ -4389,11 +4485,12 @@ _curses_tparm_impl(PyObject *module, const char *str, int i1, int i2, int i3, { char* result = NULL; - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); result = tparm((char *)str,i1,i2,i3,i4,i5,i6,i7,i8,i9); if (!result) { - PyErr_SetString(PyCursesError, "tparm() returned NULL"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "tparm() returned NULL"); return NULL; } @@ -4417,9 +4514,9 @@ static PyObject * _curses_typeahead_impl(PyObject *module, int fd) /*[clinic end generated code: output=084bb649d7066583 input=f2968d8e1805051b]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(typeahead( fd ), "typeahead"); + return PyCursesCheckERR(module, typeahead( fd ), "typeahead"); } #endif @@ -4441,7 +4538,7 @@ _curses_unctrl(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; @@ -4464,12 +4561,12 @@ _curses_ungetch(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; - return PyCursesCheckERR(ungetch(ch_), "ungetch"); + return PyCursesCheckERR(module, ungetch(ch_), "ungetch"); } #ifdef HAVE_NCURSESW @@ -4535,11 +4632,11 @@ _curses_unget_wch(PyObject *module, PyObject *ch) { wchar_t wch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; - return PyCursesCheckERR(unget_wch(wch), "unget_wch"); + return PyCursesCheckERR(module, unget_wch(wch), "unget_wch"); } #endif @@ -4587,14 +4684,15 @@ _curses_use_default_colors_impl(PyObject *module) { int code; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); code = use_default_colors(); if (code != ERR) { Py_RETURN_NONE; } else { - PyErr_SetString(PyCursesError, "use_default_colors() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "use_default_colors() returned ERR"); return NULL; } } @@ -4674,7 +4772,7 @@ _curses_has_extended_color_support_impl(PyObject *module) /* List of functions defined in the module */ -static PyMethodDef PyCurses_methods[] = { +static PyMethodDef cursesmodule_methods[] = { _CURSES_BAUDRATE_METHODDEF _CURSES_BEEP_METHODDEF _CURSES_CAN_CHANGE_COLOR_METHODDEF @@ -4759,37 +4857,146 @@ static PyMethodDef PyCurses_methods[] = { {NULL, NULL} /* sentinel */ }; -/* Initialization function for the module */ +/* Module C API */ +/* Function versions of the 3 functions for testing whether curses has been + initialised or not. */ -static struct PyModuleDef _cursesmodule = { - PyModuleDef_HEAD_INIT, - "_curses", - NULL, - -1, - PyCurses_methods, - NULL, - NULL, - NULL, - NULL -}; +static inline int +curses_capi_setupterm_called(void) +{ + return _PyCursesCheckFunction(curses_setupterm_called, "setupterm"); +} + +static inline int +curses_capi_initscr_called(void) +{ + return _PyCursesCheckFunction(curses_initscr_called, "initscr"); +} + +static inline int +curses_capi_start_color_called(void) +{ + return _PyCursesCheckFunction(curses_start_color_called, "start_color"); +} + +static void * +curses_capi_new(cursesmodule_state *state) +{ + assert(state->window_type != NULL); + void **capi = (void **)PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); + if (capi == NULL) { + PyErr_NoMemory(); + return NULL; + } + capi[0] = (void *)Py_NewRef(state->window_type); + capi[1] = curses_capi_setupterm_called; + capi[2] = curses_capi_initscr_called; + capi[3] = curses_capi_start_color_called; + return (void *)capi; +} static void -curses_destructor(PyObject *op) +curses_capi_free(void *capi) { - void *ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); - Py_DECREF(*(void **)ptr); - PyMem_Free(ptr); + assert(capi != NULL); + void **capi_ptr = (void **)capi; + // In free-threaded builds, capi_ptr[0] may have been already cleared + // by curses_capi_capsule_destructor(), hence the use of Py_XDECREF(). + Py_XDECREF(capi_ptr[0]); // decref curses window type + PyMem_Free(capi_ptr); +} + +/* Module C API Capsule */ + +static void +curses_capi_capsule_destructor(PyObject *op) +{ + void *capi = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + curses_capi_free(capi); +} + +static int +curses_capi_capsule_traverse(PyObject *op, visitproc visit, void *arg) +{ + void **capi_ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + assert(capi_ptr != NULL); + Py_VISIT(capi_ptr[0]); // visit curses window type + return 0; +} + +static int +curses_capi_capsule_clear(PyObject *op) +{ + void **capi_ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + assert(capi_ptr != NULL); + Py_CLEAR(capi_ptr[0]); // clear curses window type + return 0; +} + +static PyObject * +curses_capi_capsule_new(void *capi) +{ + PyObject *capsule = PyCapsule_New(capi, PyCurses_CAPSULE_NAME, + curses_capi_capsule_destructor); + if (capsule == NULL) { + return NULL; + } + if (_PyCapsule_SetTraverse(capsule, + curses_capi_capsule_traverse, + curses_capi_capsule_clear) < 0) + { + Py_DECREF(capsule); + return NULL; + } + return capsule; +} + +/* Module initialization and cleanup functions */ + +static int +cursesmodule_traverse(PyObject *mod, visitproc visit, void *arg) +{ + cursesmodule_state *state = get_cursesmodule_state(mod); + Py_VISIT(state->error); + Py_VISIT(state->window_type); + return 0; +} + +static int +cursesmodule_clear(PyObject *mod) +{ + cursesmodule_state *state = get_cursesmodule_state(mod); + Py_CLEAR(state->error); + Py_CLEAR(state->window_type); + return 0; +} + +static void +cursesmodule_free(void *mod) +{ + (void)cursesmodule_clear((PyObject *)mod); + curses_module_loaded = 0; // allow reloading once garbage-collected } static int cursesmodule_exec(PyObject *module) { + if (curses_module_loaded) { + PyErr_SetString(PyExc_ImportError, + "module 'curses' can only be loaded once per process"); + return -1; + } + curses_module_loaded = 1; + + cursesmodule_state *state = get_cursesmodule_state(module); /* Initialize object type */ - if (PyType_Ready(&PyCursesWindow_Type) < 0) { + state->window_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &PyCursesWindow_Type_spec, NULL); + if (state->window_type == NULL) { return -1; } - if (PyModule_AddType(module, &PyCursesWindow_Type) < 0) { + if (PyModule_AddType(module, state->window_type) < 0) { return -1; } @@ -4799,38 +5006,29 @@ cursesmodule_exec(PyObject *module) return -1; } - void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); - if (PyCurses_API == NULL) { - PyErr_NoMemory(); + /* Create the C API object */ + void *capi = curses_capi_new(state); + if (capi == NULL) { return -1; } - /* Initialize the C API pointer array */ - PyCurses_API[0] = (void *)Py_NewRef(&PyCursesWindow_Type); - PyCurses_API[1] = (void *)func_PyCursesSetupTermCalled; - PyCurses_API[2] = (void *)func_PyCursesInitialised; - PyCurses_API[3] = (void *)func_PyCursesInitialisedColor; - /* Add a capsule for the C API */ - PyObject *c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, - curses_destructor); - if (c_api_object == NULL) { - Py_DECREF(PyCurses_API[0]); - PyMem_Free(PyCurses_API); + PyObject *capi_capsule = curses_capi_capsule_new(capi); + if (capi_capsule == NULL) { + curses_capi_free(capi); return -1; } - int rc = PyDict_SetItemString(module_dict, "_C_API", c_api_object); - Py_DECREF(c_api_object); + int rc = PyDict_SetItemString(module_dict, "_C_API", capi_capsule); + Py_DECREF(capi_capsule); if (rc < 0) { return -1; } /* For exception curses.error */ - PyCursesError = PyErr_NewException("_curses.error", NULL, NULL); - if (PyCursesError == NULL) { + state->error = PyErr_NewException("_curses.error", NULL, NULL); + if (state->error == NULL) { return -1; } - rc = PyDict_SetItemString(module_dict, "error", PyCursesError); - Py_DECREF(PyCursesError); + rc = PyDict_SetItemString(module_dict, "error", state->error); if (rc < 0) { return -1; } @@ -5022,26 +5220,28 @@ cursesmodule_exec(PyObject *module) return 0; } +/* Initialization function for the module */ + +static PyModuleDef_Slot cursesmodule_slots[] = { + {Py_mod_exec, cursesmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +static struct PyModuleDef cursesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_curses", + .m_size = sizeof(cursesmodule_state), + .m_methods = cursesmodule_methods, + .m_slots = cursesmodule_slots, + .m_traverse = cursesmodule_traverse, + .m_clear = cursesmodule_clear, + .m_free = cursesmodule_free +}; + PyMODINIT_FUNC PyInit__curses(void) { - // create the module - PyObject *mod = PyModule_Create(&_cursesmodule); - if (mod == NULL) { - goto error; - } -#ifdef Py_GIL_DISABLED - if (PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED) < 0) { - goto error; - } -#endif - // populate the module - if (cursesmodule_exec(mod) < 0) { - goto error; - } - return mod; - -error: - Py_XDECREF(mod); - return NULL; + return PyModuleDef_Init(&cursesmodule); } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8562e0ca0bbbab..2ba46cddb4f558 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2921,7 +2921,7 @@ delta_bool(PyDateTime_Delta *self) static PyObject * delta_repr(PyDateTime_Delta *self) { - PyObject *args = PyUnicode_FromString(""); + PyObject *args = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (args == NULL) { return NULL; @@ -3445,6 +3445,27 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return new_date_subclass_ex(year, month, day, cls); } +/* Return new date from _strptime.strptime_datetime_date(). */ +static PyObject * +date_strptime(PyObject *cls, PyObject *args) +{ + PyObject *string, *format, *result; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { + return NULL; + } + + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); + if (module == NULL) { + return NULL; + } + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_date), cls, + string, format, NULL); + Py_DECREF(module); + return result; +} + /* * Date arithmetic. @@ -3910,6 +3931,11 @@ static PyMethodDef date_methods[] = { "number and weekday.\n\n" "This is the inverse of the date.isocalendar() function")}, + {"strptime", (PyCFunction)date_strptime, + METH_VARARGS | METH_CLASS, + PyDoc_STR("string, format -> new date parsed from a string " + "(like time.strptime()).")}, + {"today", (PyCFunction)date_today, METH_NOARGS | METH_CLASS, PyDoc_STR("Current date or datetime: same as " "self.__class__.fromtimestamp(time.time()).")}, @@ -4644,6 +4670,27 @@ time_new(PyTypeObject *type, PyObject *args, PyObject *kw) return self; } +/* Return new time from _strptime.strptime_datetime_time(). */ +static PyObject * +time_strptime(PyObject *cls, PyObject *args) +{ + PyObject *string, *format, *result; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { + return NULL; + } + + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); + if (module == NULL) { + return NULL; + } + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_time), cls, + string, format, NULL); + Py_DECREF(module); + return result; +} + /* * Destructor. */ @@ -4997,6 +5044,14 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { goto invalid_string_error; } + if (hour == 24) { + if (minute == 0 && second == 0 && microsecond == 0) { + hour = 0; + } else { + goto invalid_iso_midnight; + } + } + PyObject *tzinfo = tzinfo_from_isoformat_results(rv, tzoffset, tzimicrosecond); @@ -5015,6 +5070,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { Py_DECREF(tzinfo); return t; +invalid_iso_midnight: + PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + return NULL; + invalid_string_error: PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", tstr); return NULL; @@ -5067,6 +5126,15 @@ time_reduce(PyDateTime_Time *self, PyObject *arg) static PyMethodDef time_methods[] = { + /* Class method: */ + + {"strptime", (PyCFunction)time_strptime, + METH_VARARGS | METH_CLASS, + PyDoc_STR("string, format -> new time parsed from a string " + "(like time.strptime()).")}, + + /* Instance methods: */ + {"isoformat", _PyCFunction_CAST(time_isoformat), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" "[+HH:MM].\n\n" @@ -5574,7 +5642,7 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) return result; } -/* Return new datetime from _strptime.strptime_datetime(). */ +/* Return new datetime from _strptime.strptime_datetime_datetime(). */ static PyObject * datetime_strptime(PyObject *cls, PyObject *args) { @@ -5587,7 +5655,8 @@ datetime_strptime(PyObject *cls, PyObject *args) if (module == NULL) { return NULL; } - result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_datetime), cls, string, format, NULL); Py_DECREF(module); return result; @@ -5861,6 +5930,26 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) goto error; } + if ((hour == 24) && (month <= 12)) { + int d_in_month = days_in_month(year, month); + if (day <= d_in_month) { + if (minute == 0 && second == 0 && microsecond == 0) { + // Calculate midnight of the next day + hour = 0; + day += 1; + if (day > d_in_month) { + day = 1; + month += 1; + if (month > 12) { + month = 1; + year += 1; + } + } + } else { + goto invalid_iso_midnight; + } + } + } PyObject *dt = new_datetime_subclass_ex(year, month, day, hour, minute, second, microsecond, tzinfo, cls); @@ -5868,6 +5957,12 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) Py_DECREF(dtstr_clean); return dt; +invalid_iso_midnight: + PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + Py_DECREF(tzinfo); + Py_DECREF(dtstr_clean); + return NULL; + invalid_string_error: PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e99a96ab93392e..c564813036e504 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -122,6 +122,8 @@ get_module_state(PyObject *mod) } static struct PyModuleDef _decimal_module; +static PyType_Spec dec_spec; +static PyType_Spec context_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -134,10 +136,16 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = _PyType_GetModuleByDef2(Py_TYPE(left), Py_TYPE(right), - &_decimal_module); - assert(mod != NULL); - return get_module_state(mod); + PyTypeObject *base; + if (PyType_GetBaseByToken(Py_TYPE(left), &dec_spec, &base) != 1) { + assert(!PyErr_Occurred()); + PyType_GetBaseByToken(Py_TYPE(right), &dec_spec, &base); + } + assert(base != NULL); + void *state = _PyType_GetModuleState(base); + assert(state != NULL); + Py_DECREF(base); + return (decimal_state *)state; } @@ -183,6 +191,7 @@ typedef struct PyDecContextObject { PyObject *flags; int capitals; PyThreadState *tstate; + decimal_state *modstate; } PyDecContextObject; typedef struct { @@ -203,6 +212,15 @@ typedef struct { #define CTX(v) (&((PyDecContextObject *)v)->ctx) #define CtxCaps(v) (((PyDecContextObject *)v)->capitals) +static inline decimal_state * +get_module_state_from_ctx(PyObject *v) +{ + assert(PyType_GetBaseByToken(Py_TYPE(v), &context_spec, NULL) == 1); + decimal_state *state = ((PyDecContextObject *)v)->modstate; + assert(state != NULL); + return state; +} + Py_LOCAL_INLINE(PyObject *) incr_true(void) @@ -557,7 +575,7 @@ static int dec_addstatus(PyObject *context, uint32_t status) { mpd_context_t *ctx = CTX(context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); ctx->status |= status; if (status & (ctx->traps|MPD_Malloc_error)) { @@ -745,7 +763,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) { PyObject *res = Py_NotImplemented; - decimal_state *state = find_state_left_or_right(v, w); + decimal_state *state = get_module_state_by_def(Py_TYPE(v)); assert(PyDecSignalDict_Check(state, v)); if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { @@ -852,7 +870,7 @@ static PyObject * context_getround(PyObject *self, void *Py_UNUSED(closure)) { int i = mpd_getround(CTX(self)); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); return Py_NewRef(state->round_map[i]); } @@ -1011,7 +1029,7 @@ context_setround(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) mpd_context_t *ctx; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); x = getround(state, value); if (x == -1) { return -1; @@ -1070,7 +1088,7 @@ context_settraps_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { return -1; @@ -1090,7 +1108,7 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1135,7 +1153,7 @@ context_setstatus_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { @@ -1156,7 +1174,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1386,6 +1404,7 @@ context_new(PyTypeObject *type, CtxCaps(self) = 1; self->tstate = NULL; + self->modstate = state; if (type == state->PyDecContext_Type) { PyObject_GC_Track(self); @@ -1463,7 +1482,7 @@ context_repr(PyDecContextObject *self) int n, mem; #ifdef Py_DEBUG - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx((PyObject *)self); assert(PyDecContext_Check(state, self)); #endif ctx = CTX(self); @@ -1554,7 +1573,7 @@ context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *copy; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; @@ -1574,7 +1593,7 @@ context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); ctx = CTX(self); @@ -2015,11 +2034,10 @@ static PyType_Spec ctxmanager_spec = { /******************************************************************************/ static PyObject * -PyDecType_New(PyTypeObject *type) +PyDecType_New(decimal_state *state, PyTypeObject *type) { PyDecObject *dec; - decimal_state *state = get_module_state_by_def(type); if (type == state->PyDec_Type) { dec = PyObject_GC_New(PyDecObject, state->PyDec_Type); } @@ -2045,7 +2063,7 @@ PyDecType_New(PyTypeObject *type) assert(PyObject_GC_IsTracked((PyObject *)dec)); return (PyObject *)dec; } -#define dec_alloc(st) PyDecType_New((st)->PyDec_Type) +#define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) static int dec_traverse(PyObject *dec, visitproc visit, void *arg) @@ -2148,7 +2166,8 @@ PyDecType_FromCString(PyTypeObject *type, const char *s, PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2172,7 +2191,8 @@ PyDecType_FromCStringExact(PyTypeObject *type, const char *s, uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2259,7 +2279,8 @@ PyDecType_FromSsize(PyTypeObject *type, mpd_ssize_t v, PyObject *context) PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2280,7 +2301,8 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2298,13 +2320,13 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) /* Convert from a PyLongObject. The context is not modified; flags set during conversion are accumulated in the status parameter. */ static PyObject * -dec_from_long(PyTypeObject *type, PyObject *v, +dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { PyObject *dec; PyLongObject *l = (PyLongObject *)v; - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2349,7 +2371,8 @@ PyDecType_FromLong(PyTypeObject *type, PyObject *v, PyObject *context) return NULL; } - dec = dec_from_long(type, v, CTX(context), &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2378,7 +2401,8 @@ PyDecType_FromLongExact(PyTypeObject *type, PyObject *v, } mpd_maxcontext(&maxctx); - dec = dec_from_long(type, v, &maxctx, &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, &maxctx, &status); if (dec == NULL) { return NULL; } @@ -2410,7 +2434,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, mpd_t *d1, *d2; uint32_t status = 0; mpd_context_t maxctx; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); #ifdef Py_DEBUG assert(PyType_IsSubtype(type, state->PyDec_Type)); @@ -2431,7 +2455,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, sign = (copysign(1.0, x) == 1.0) ? 0 : 1; if (isnan(x) || isinf(x)) { - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2548,12 +2572,12 @@ PyDecType_FromDecimalExact(PyTypeObject *type, PyObject *v, PyObject *context) PyObject *dec; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (type == state->PyDec_Type && PyDec_CheckExact(state, v)) { return Py_NewRef(v); } - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2833,11 +2857,56 @@ dec_from_float(PyObject *type, PyObject *pyfloat) return result; } +/* 'v' can have any numeric type accepted by the Decimal constructor. Attempt + an exact conversion. If the result does not meet the restrictions + for an mpd_t, fail with InvalidOperation. */ +static PyObject * +PyDecType_FromNumberExact(PyTypeObject *type, PyObject *v, PyObject *context) +{ + decimal_state *state = get_module_state_by_def(type); + assert(v != NULL); + if (PyDec_Check(state, v)) { + return PyDecType_FromDecimalExact(type, v, context); + } + else if (PyLong_Check(v)) { + return PyDecType_FromLongExact(type, v, context); + } + else if (PyFloat_Check(v)) { + if (dec_addstatus(context, MPD_Float_operation)) { + return NULL; + } + return PyDecType_FromFloatExact(type, v, context); + } + else { + PyErr_Format(PyExc_TypeError, + "conversion from %s to Decimal is not supported", + Py_TYPE(v)->tp_name); + return NULL; + } +} + +/* class method */ +static PyObject * +dec_from_number(PyObject *type, PyObject *number) +{ + PyObject *context; + PyObject *result; + + decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + CURRENT_CONTEXT(state, context); + result = PyDecType_FromNumberExact(state->PyDec_Type, number, context); + if (type != (PyObject *)state->PyDec_Type && result != NULL) { + Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + } + + return result; +} + /* create_decimal_from_float */ static PyObject * ctx_from_float(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); return PyDec_FromFloat(state, v, context); } @@ -2848,7 +2917,7 @@ dec_apply(PyObject *v, PyObject *context) PyObject *result; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -2875,7 +2944,7 @@ dec_apply(PyObject *v, PyObject *context) static PyObject * PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDecType_FromSsizeExact(type, 0, context); } @@ -2910,7 +2979,7 @@ PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) static PyObject * PyDec_FromObject(PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDec_FromSsize(state, 0, context); } @@ -2997,7 +3066,7 @@ ctx_create_decimal(PyObject *context, PyObject *args) Py_LOCAL_INLINE(int) convert_op(int type_err, PyObject **conv, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, v)) { *conv = Py_NewRef(v); return 1; @@ -3100,7 +3169,7 @@ multiply_by_denominator(PyObject *v, PyObject *r, PyObject *context) if (tmp == NULL) { return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); denom = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); if (denom == NULL) { @@ -3155,7 +3224,7 @@ numerator_as_decimal(PyObject *r, PyObject *context) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); num = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); return num; @@ -3174,7 +3243,7 @@ convert_op_cmp(PyObject **vcmp, PyObject **wcmp, PyObject *v, PyObject *w, *vcmp = v; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, w)) { *wcmp = Py_NewRef(w); } @@ -4414,12 +4483,11 @@ dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) return Py_NewRef(self); } -static PyObject * -dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +static inline PyObject * +_dec_mpd_radix(decimal_state *state) { PyObject *result; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -4429,6 +4497,13 @@ dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +static PyObject * +dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +{ + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + return _dec_mpd_radix(state); +} + static PyObject * dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) { @@ -5022,6 +5097,7 @@ static PyMethodDef dec_methods [] = /* Miscellaneous */ { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, + { "from_number", dec_from_number, METH_O|METH_CLASS, doc_from_number }, { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, @@ -5041,6 +5117,7 @@ static PyMethodDef dec_methods [] = }; static PyType_Slot dec_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, dec_traverse}, @@ -5130,7 +5207,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ \ CONVERT_OP_RAISE(&a, v, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5162,7 +5239,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5198,7 +5275,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5227,7 +5304,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ } \ \ CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); \ + decimal_state *state = get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5293,7 +5370,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { Py_DECREF(a); @@ -5348,7 +5425,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5383,7 +5460,8 @@ DecCtx_TernaryFunc(mpd_qfma) static PyObject * ctx_mpd_radix(PyObject *context, PyObject *dummy) { - return dec_mpd_radix(context, dummy); + decimal_state *state = get_module_state_from_ctx(context); + return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ @@ -5400,7 +5478,7 @@ DecCtx_BoolFunc_NO_CTX(mpd_iszero) static PyObject * ctx_iscanonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5426,7 +5504,7 @@ PyDecContext_Apply(PyObject *context, PyObject *v) static PyObject * ctx_canonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5443,7 +5521,7 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5476,7 +5554,7 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5573,7 +5651,7 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5728,6 +5806,7 @@ static PyMethodDef context_methods [] = }; static PyType_Slot context_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, context_dealloc}, {Py_tp_traverse, context_traverse}, {Py_tp_clear, context_clear}, diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index a1823cdd32b74c..b34bff83d3f4e9 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -189,6 +189,19 @@ Decimal.from_float(0.1) is not the same as Decimal('0.1').\n\ \n\ \n"); +PyDoc_STRVAR(doc_from_number, +"from_number($type, number, /)\n--\n\n\ +Class method that converts a real number to a decimal number, exactly.\n\ +\n\ + >>> Decimal.from_number(314) # int\n\ + Decimal('314')\n\ + >>> Decimal.from_number(0.1) # float\n\ + Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n\ + Decimal('3.14')\n\ +\n\ +\n"); + PyDoc_STRVAR(doc_fma, "fma($self, /, other, third, context=None)\n--\n\n\ Fused multiply-add. Return self*other+third with no rounding of the\n\ diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index ec999582d2fb9d..e134e096e044b7 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -196,7 +196,7 @@ list_join(PyObject* list) PyObject* joiner; PyObject* result; - joiner = PyUnicode_FromStringAndSize("", 0); + joiner = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (!joiner) return NULL; result = PyUnicode_Join(joiner, list); @@ -1317,7 +1317,7 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject* text = element_get_text((ElementObject*)item); if (text == Py_None) { Py_DECREF(item); - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } Py_XINCREF(text); Py_DECREF(item); diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 64766b474514bf..802b1cf792c555 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -25,6 +25,8 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; + PyTypeObject *placeholder_type; + PyObject *placeholder; // strong reference (singleton) PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -41,6 +43,95 @@ get_functools_state(PyObject *module) /* partial object **********************************************************/ + +// The 'Placeholder' singleton indicates which formal positional +// parameters are to be bound first when using a 'partial' object. + +typedef struct { + PyObject_HEAD +} placeholderobject; + +static inline _functools_state * +get_functools_state_by_type(PyTypeObject *type); + +PyDoc_STRVAR(placeholder_doc, +"The type of the Placeholder singleton.\n\n" +"Used as a placeholder for partial arguments."); + +static PyObject * +placeholder_repr(PyObject *op) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyObject * +placeholder_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyMethodDef placeholder_methods[] = { + {"__reduce__", placeholder_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static void +placeholder_dealloc(PyObject* self) +{ + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); +} + +static PyObject * +placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); + return NULL; + } + _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder != NULL) { + return Py_NewRef(state->placeholder); + } + + PyObject *placeholder = PyType_GenericNew(type, NULL, NULL); + if (placeholder == NULL) { + return NULL; + } + + if (state->placeholder == NULL) { + state->placeholder = Py_NewRef(placeholder); + } + return placeholder; +} + +static int +placeholder_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static PyType_Slot placeholder_type_slots[] = { + {Py_tp_dealloc, placeholder_dealloc}, + {Py_tp_repr, placeholder_repr}, + {Py_tp_doc, (void *)placeholder_doc}, + {Py_tp_methods, placeholder_methods}, + {Py_tp_new, placeholder_new}, + {Py_tp_traverse, placeholder_traverse}, + {0, 0} +}; + +static PyType_Spec placeholder_type_spec = { + .name = "functools._PlaceholderType", + .basicsize = sizeof(placeholderobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, + .slots = placeholder_type_slots +}; + + typedef struct { PyObject_HEAD PyObject *fn; @@ -48,13 +139,18 @@ typedef struct { PyObject *kw; PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ + PyObject *placeholder; /* Placeholder for positional arguments */ + Py_ssize_t phcount; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; +// cast a PyObject pointer PTR to a partialobject pointer (no type checks) +#define _PyPartialObject_CAST(PTR) ((partialobject *)(PTR)) + static void partial_setvectorcall(partialobject *pto); static struct PyModuleDef _functools_module; static PyObject * -partial_call(partialobject *pto, PyObject *args, PyObject *kwargs); +partial_call(PyObject *pto, PyObject *args, PyObject *kwargs); static inline _functools_state * get_functools_state_by_type(PyTypeObject *type) @@ -70,23 +166,38 @@ get_functools_state_by_type(PyTypeObject *type) static PyObject * partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *func, *pargs, *nargs, *pkw; + PyObject *func, *pto_args, *new_args, *pto_kw, *phold; partialobject *pto; + Py_ssize_t pto_phcount = 0; + Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1; - if (PyTuple_GET_SIZE(args) < 1) { + if (new_nargs < 0) { PyErr_SetString(PyExc_TypeError, "type 'partial' takes at least one argument"); return NULL; } + func = PyTuple_GET_ITEM(args, 0); + if (!PyCallable_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "the first argument must be callable"); + return NULL; + } _functools_state *state = get_functools_state_by_type(type); if (state == NULL) { return NULL; } + phold = state->placeholder; - pargs = pkw = NULL; - func = PyTuple_GET_ITEM(args, 0); + /* Placeholder restrictions */ + if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + /* check wrapped function / object */ + pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); if (res == -1) { return NULL; @@ -95,18 +206,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) // We can use its underlying function directly and merge the arguments. partialobject *part = (partialobject *)func; if (part->dict == NULL) { - pargs = part->args; - pkw = part->kw; + pto_args = part->args; + pto_kw = part->kw; func = part->fn; - assert(PyTuple_Check(pargs)); - assert(PyDict_Check(pkw)); + pto_phcount = part->phcount; + assert(PyTuple_Check(pto_args)); + assert(PyDict_Check(pto_kw)); } } - if (!PyCallable_Check(func)) { - PyErr_SetString(PyExc_TypeError, - "the first argument must be callable"); - return NULL; - } /* create partialobject structure */ pto = (partialobject *)type->tp_alloc(type, 0); @@ -114,18 +221,58 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; pto->fn = Py_NewRef(func); + pto->placeholder = phold; - nargs = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX); - if (nargs == NULL) { + new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); + if (new_args == NULL) { Py_DECREF(pto); return NULL; } - if (pargs == NULL) { - pto->args = nargs; + + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + if (PyTuple_GET_ITEM(new_args, i) == phold) { + phcount++; + } + } + /* merge args with args of `func` which is `partial` */ + if (pto_phcount > 0 && new_nargs > 0) { + Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); + Py_ssize_t tot_nargs = npargs; + if (new_nargs > pto_phcount) { + tot_nargs += new_nargs - pto_phcount; + } + PyObject *item; + PyObject *tot_args = PyTuple_New(tot_nargs); + for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { + if (i < npargs) { + item = PyTuple_GET_ITEM(pto_args, i); + if (j < new_nargs && item == phold) { + item = PyTuple_GET_ITEM(new_args, j); + j++; + pto_phcount--; + } + } + else { + item = PyTuple_GET_ITEM(new_args, j); + j++; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + pto->args = tot_args; + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); + } + else if (pto_args == NULL) { + pto->args = new_args; + pto->phcount = phcount; } else { - pto->args = PySequence_Concat(pargs, nargs); - Py_DECREF(nargs); + pto->args = PySequence_Concat(pto_args, new_args); + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); if (pto->args == NULL) { Py_DECREF(pto); return NULL; @@ -133,7 +280,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) assert(PyTuple_Check(pto->args)); } - if (pkw == NULL || PyDict_GET_SIZE(pkw) == 0) { + if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) { if (kw == NULL) { pto->kw = PyDict_New(); } @@ -145,7 +292,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } } else { - pto->kw = PyDict_Copy(pkw); + pto->kw = PyDict_Copy(pto_kw); if (kw != NULL && pto->kw != NULL) { if (PyDict_Merge(pto->kw, kw, 1) != 0) { Py_DECREF(pto); @@ -163,8 +310,9 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } static int -partial_clear(partialobject *pto) +partial_clear(PyObject *self) { + partialobject *pto = _PyPartialObject_CAST(self); Py_CLEAR(pto->fn); Py_CLEAR(pto->args); Py_CLEAR(pto->kw); @@ -173,8 +321,9 @@ partial_clear(partialobject *pto) } static int -partial_traverse(partialobject *pto, visitproc visit, void *arg) +partial_traverse(PyObject *self, visitproc visit, void *arg) { + partialobject *pto = _PyPartialObject_CAST(self); Py_VISIT(Py_TYPE(pto)); Py_VISIT(pto->fn); Py_VISIT(pto->args); @@ -184,16 +333,16 @@ partial_traverse(partialobject *pto, visitproc visit, void *arg) } static void -partial_dealloc(partialobject *pto) +partial_dealloc(PyObject *self) { - PyTypeObject *tp = Py_TYPE(pto); + PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(pto); - if (pto->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) pto); + PyObject_GC_UnTrack(self); + if (_PyPartialObject_CAST(self)->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } - (void)partial_clear(pto); - tp->tp_free(pto); + (void)partial_clear(self); + tp->tp_free(self); Py_DECREF(tp); } @@ -216,32 +365,39 @@ partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, { pto->vectorcall = NULL; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, (PyObject *)pto, - args, nargs, kwnames); + return _PyObject_MakeTpCall(tstate, (PyObject *)pto, args, nargs, kwnames); } static PyObject * -partial_vectorcall(partialobject *pto, PyObject *const *args, +partial_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + partialobject *pto = _PyPartialObject_CAST(self);; PyThreadState *tstate = _PyThreadState_GET(); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); /* pto->kw is mutable, so need to check every time */ if (PyDict_GET_SIZE(pto->kw)) { return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); } + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", pto_phcount, nargs); + return NULL; + } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - Py_ssize_t nargs_total = nargs; + Py_ssize_t nargskw = nargs; if (kwnames != NULL) { - nargs_total += PyTuple_GET_SIZE(kwnames); + nargskw += PyTuple_GET_SIZE(kwnames); } PyObject **pto_args = _PyTuple_ITEMS(pto->args); Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); /* Fast path if we're called without arguments */ - if (nargs_total == 0) { + if (nargskw == 0) { return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, pto_nargs, NULL); } @@ -258,29 +414,47 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, return ret; } - Py_ssize_t newnargs_total = pto_nargs + nargs_total; - PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject *ret; PyObject **stack; - if (newnargs_total <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount; + if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc(newnargs_total * sizeof(PyObject *)); + stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; } } - /* Copy to new stack, using borrowed references */ - memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); - - ret = _PyObject_VectorcallTstate(tstate, pto->fn, - stack, pto_nargs + nargs, kwnames); + Py_ssize_t tot_nargs; + if (pto_phcount) { + tot_nargs = pto_nargs + nargs - pto_phcount; + Py_ssize_t j = 0; // New args index + for (Py_ssize_t i = 0; i < pto_nargs; i++) { + if (pto_args[i] == pto->placeholder) { + stack[i] = args[j]; + j += 1; + } + else { + stack[i] = pto_args[i]; + } + } + assert(j == pto_phcount); + if (nargskw > pto_phcount) { + memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*)); + } + } + else { + tot_nargs = pto_nargs + nargs; + /* Copy to new stack, using borrowed references */ + memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*)); + } + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, + stack, tot_nargs, kwnames); if (stack != small_stack) { PyMem_Free(stack); } @@ -299,53 +473,95 @@ partial_setvectorcall(partialobject *pto) * but that is unlikely (why use partial without arguments?), * so we don't optimize that */ else { - pto->vectorcall = (vectorcallfunc)partial_vectorcall; + pto->vectorcall = partial_vectorcall; } } // Not converted to argument clinic, because of `*args, **kwargs` arguments. static PyObject * -partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) +partial_call(PyObject *self, PyObject *args, PyObject *kwargs) { + partialobject *pto = _PyPartialObject_CAST(self); assert(PyCallable_Check(pto->fn)); assert(PyTuple_Check(pto->args)); assert(PyDict_Check(pto->kw)); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", pto_phcount, nargs); + return NULL; + } + /* Merge keywords */ - PyObject *kwargs2; + PyObject *tot_kw; if (PyDict_GET_SIZE(pto->kw) == 0) { /* kwargs can be NULL */ - kwargs2 = Py_XNewRef(kwargs); + tot_kw = Py_XNewRef(kwargs); } else { /* bpo-27840, bpo-29318: dictionary of keyword parameters must be copied, because a function using "**kwargs" can modify the dictionary. */ - kwargs2 = PyDict_Copy(pto->kw); - if (kwargs2 == NULL) { + tot_kw = PyDict_Copy(pto->kw); + if (tot_kw == NULL) { return NULL; } if (kwargs != NULL) { - if (PyDict_Merge(kwargs2, kwargs, 1) != 0) { - Py_DECREF(kwargs2); + if (PyDict_Merge(tot_kw, kwargs, 1) != 0) { + Py_DECREF(tot_kw); return NULL; } } } /* Merge positional arguments */ - /* Note: tupleconcat() is optimized for empty tuples */ - PyObject *args2 = PySequence_Concat(pto->args, args); - if (args2 == NULL) { - Py_XDECREF(kwargs2); - return NULL; + PyObject *tot_args; + if (pto_phcount) { + Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); + Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + assert(tot_nargs >= 0); + tot_args = PyTuple_New(tot_nargs); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); + return NULL; + } + PyObject *pto_args = pto->args; + PyObject *item; + Py_ssize_t j = 0; // New args index + for (Py_ssize_t i = 0; i < pto_nargs; i++) { + item = PyTuple_GET_ITEM(pto_args, i); + if (item == pto->placeholder) { + item = PyTuple_GET_ITEM(args, j); + j += 1; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + assert(j == pto_phcount); + for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) { + item = PyTuple_GET_ITEM(args, j); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + j += 1; + } + } + else { + /* Note: tupleconcat() is optimized for empty tuples */ + tot_args = PySequence_Concat(pto->args, args); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); + return NULL; + } } - PyObject *res = PyObject_Call(pto->fn, args2, kwargs2); - Py_DECREF(args2); - Py_XDECREF(kwargs2); + PyObject *res = PyObject_Call(pto->fn, tot_args, tot_kw); + Py_DECREF(tot_args); + Py_XDECREF(tot_kw); return res; } @@ -377,8 +593,9 @@ static PyGetSetDef partial_getsetlist[] = { }; static PyObject * -partial_repr(partialobject *pto) +partial_repr(PyObject *self) { + partialobject *pto = _PyPartialObject_CAST(self); PyObject *result = NULL; PyObject *arglist; PyObject *mod; @@ -387,18 +604,18 @@ partial_repr(partialobject *pto) PyObject *key, *value; int status; - status = Py_ReprEnter((PyObject *)pto); + status = Py_ReprEnter(self); if (status != 0) { if (status < 0) return NULL; return PyUnicode_FromString("..."); } - arglist = PyUnicode_FromString(""); + arglist = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (arglist == NULL) goto done; /* Pack positional arguments */ - assert (PyTuple_Check(pto->args)); + assert(PyTuple_Check(pto->args)); n = PyTuple_GET_SIZE(pto->args); for (i = 0; i < n; i++) { Py_SETREF(arglist, PyUnicode_FromFormat("%U, %R", arglist, @@ -433,11 +650,11 @@ partial_repr(partialobject *pto) Py_DECREF(arglist); done: - Py_ReprLeave((PyObject *)pto); + Py_ReprLeave(self); return result; error: Py_DECREF(arglist); - Py_ReprLeave((PyObject *)pto); + Py_ReprLeave(self); return NULL; } @@ -449,20 +666,25 @@ partial_repr(partialobject *pto) */ static PyObject * -partial_reduce(partialobject *pto, PyObject *unused) +partial_reduce(PyObject *self, PyObject *Py_UNUSED(args)) { + partialobject *pto = _PyPartialObject_CAST(self); return Py_BuildValue("O(O)(OOOO)", Py_TYPE(pto), pto->fn, pto->fn, pto->args, pto->kw, pto->dict ? pto->dict : Py_None); } static PyObject * -partial_setstate(partialobject *pto, PyObject *state) +partial_setstate(PyObject *self, PyObject *state) { + partialobject *pto = _PyPartialObject_CAST(self); PyObject *fn, *fnargs, *kw, *dict; - if (!PyTuple_Check(state) || - !PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "invalid partial state"); + return NULL; + } + if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || (kw != Py_None && !PyDict_Check(kw))) @@ -471,6 +693,20 @@ partial_setstate(partialobject *pto, PyObject *state) return NULL; } + Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); + if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < nargs - 1; i++) { + if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) { + phcount++; + } + } + if(!PyTuple_CheckExact(fnargs)) fnargs = PySequence_Tuple(fnargs); else @@ -493,18 +729,18 @@ partial_setstate(partialobject *pto, PyObject *state) dict = NULL; else Py_INCREF(dict); - Py_SETREF(pto->fn, Py_NewRef(fn)); Py_SETREF(pto->args, fnargs); Py_SETREF(pto->kw, kw); + pto->phcount = phcount; Py_XSETREF(pto->dict, dict); partial_setvectorcall(pto); Py_RETURN_NONE; } static PyMethodDef partial_methods[] = { - {"__reduce__", (PyCFunction)partial_reduce, METH_NOARGS}, - {"__setstate__", (PyCFunction)partial_setstate, METH_O}, + {"__reduce__", partial_reduce, METH_NOARGS}, + {"__setstate__", partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -522,7 +758,7 @@ static PyType_Slot partial_type_slots[] = { {Py_tp_methods, partial_methods}, {Py_tp_members, partial_memberlist}, {Py_tp_getset, partial_getsetlist}, - {Py_tp_descr_get, (descrgetfunc)partial_descr_get}, + {Py_tp_descr_get, partial_descr_get}, {Py_tp_new, partial_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0} @@ -1498,6 +1734,25 @@ _functools_exec(PyObject *module) return -1; } + state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &placeholder_type_spec, NULL); + if (state->placeholder_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->placeholder_type) < 0) { + return -1; + } + + PyObject *placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (placeholder == NULL) { + return -1; + } + if (PyModule_AddObjectRef(module, "Placeholder", placeholder) < 0) { + Py_DECREF(placeholder); + return -1; + } + Py_DECREF(placeholder); + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1542,6 +1797,8 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); + Py_VISIT(state->placeholder_type); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1553,6 +1810,8 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); + Py_CLEAR(state->placeholder_type); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 1238e6074246d0..6622f2cabb908b 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -346,7 +346,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, /* buffering */ if (buffering < 0) { - PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); + PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(_isatty_open_only)); if (res == NULL) goto error; isatty = PyObject_IsTrue(res); diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8dae465fd20f8b..f374592eb95967 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -12,9 +12,6 @@ #ifdef HAVE_SYS_TYPES_H # include #endif -#ifdef HAVE_SYS_STAT_H -# include -#endif #ifdef HAVE_IO_H # include #endif @@ -86,6 +83,7 @@ typedef struct { } fileio; #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type)) +#define _PyFileIO_CAST(op) _Py_CAST(fileio*, (op)) /* Forward declarations */ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error); @@ -93,15 +91,16 @@ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool int _PyFileIO_closed(PyObject *self) { - return ((fileio *)self)->fd < 0; + return (_PyFileIO_CAST(self)->fd < 0); } /* Because this can call arbitrary code, it shouldn't be called when the refcount is 0 (that is, not directly from tp_dealloc unless the refcount has been temporarily re-incremented). */ static PyObject * -fileio_dealloc_warn(fileio *self, PyObject *source) +fileio_dealloc_warn(PyObject *op, PyObject *source) { + fileio *self = _PyFileIO_CAST(op); if (self->fd >= 0 && self->closefd) { PyObject *exc = PyErr_GetRaisedException(); if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { @@ -171,7 +170,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) exc = PyErr_GetRaisedException(); } if (self->finalizing) { - PyObject *r = fileio_dealloc_warn(self, (PyObject *) self); + PyObject *r = fileio_dealloc_warn((PyObject*)self, (PyObject *) self); if (r) { Py_DECREF(r); } @@ -192,23 +191,22 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) static PyObject * fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - fileio *self; - assert(type != NULL && type->tp_alloc != NULL); - self = (fileio *) type->tp_alloc(type, 0); - if (self != NULL) { - self->fd = -1; - self->created = 0; - self->readable = 0; - self->writable = 0; - self->appending = 0; - self->seekable = -1; - self->stat_atopen = NULL; - self->closefd = 1; - self->weakreflist = NULL; + fileio *self = (fileio *) type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } + self->fd = -1; + self->created = 0; + self->readable = 0; + self->writable = 0; + self->appending = 0; + self->seekable = -1; + self->stat_atopen = NULL; + self->closefd = 1; + self->weakreflist = NULL; return (PyObject *) self; } @@ -539,36 +537,43 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, } static int -fileio_traverse(fileio *self, visitproc visit, void *arg) +fileio_traverse(PyObject *op, visitproc visit, void *arg) { + fileio *self = _PyFileIO_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -fileio_clear(fileio *self) +fileio_clear(PyObject *op) { + fileio *self = _PyFileIO_CAST(op); Py_CLEAR(self->dict); return 0; } static void -fileio_dealloc(fileio *self) +fileio_dealloc(PyObject *op) { - PyTypeObject *tp = Py_TYPE(self); + fileio *self = _PyFileIO_CAST(op); self->finalizing = 1; - if (_PyIOBase_finalize((PyObject *) self) < 0) + if (_PyIOBase_finalize(op) < 0) { return; + } + _PyObject_GC_UNTRACK(self); if (self->stat_atopen != NULL) { PyMem_Free(self->stat_atopen); self->stat_atopen = NULL; } - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - (void)fileio_clear(self); - tp->tp_free((PyObject *)self); + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs(op); + } + (void)fileio_clear(op); + + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); Py_DECREF(tp); } @@ -1151,18 +1156,20 @@ mode_string(fileio *self) } static PyObject * -fileio_repr(fileio *self) +fileio_repr(PyObject *op) { - PyObject *nameobj, *res; - const char *type_name = Py_TYPE((PyObject *) self)->tp_name; + fileio *self = _PyFileIO_CAST(op); + const char *type_name = Py_TYPE(self)->tp_name; if (self->fd < 0) { return PyUnicode_FromFormat("<%.100s [closed]>", type_name); } + PyObject *nameobj; if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) { return NULL; } + PyObject *res; if (nameobj == NULL) { res = PyUnicode_FromFormat( "<%.100s fd=%d mode='%s' closefd=%s>", @@ -1208,6 +1215,24 @@ _io_FileIO_isatty_impl(fileio *self) return PyBool_FromLong(res); } +/* Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). */ +static PyObject * +_io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + fileio *self = _PyFileIO_CAST(op); + if (self->stat_atopen != NULL && !S_ISCHR(self->stat_atopen->st_mode)) { + Py_RETURN_FALSE; + } + return _io_FileIO_isatty_impl(self); +} + #include "clinic/fileio.c.h" static PyMethodDef fileio_methods[] = { @@ -1224,7 +1249,8 @@ static PyMethodDef fileio_methods[] = { _IO_FILEIO_WRITABLE_METHODDEF _IO_FILEIO_FILENO_METHODDEF _IO_FILEIO_ISATTY_METHODDEF - {"_dealloc_warn", (PyCFunction)fileio_dealloc_warn, METH_O, NULL}, + {"_isatty_open_only", _io_FileIO_isatty_open_only, METH_NOARGS}, + {"_dealloc_warn", fileio_dealloc_warn, METH_O, NULL}, {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} /* sentinel */ @@ -1233,26 +1259,30 @@ static PyMethodDef fileio_methods[] = { /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ static PyObject * -get_closed(fileio *self, void *closure) +fileio_get_closed(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyBool_FromLong((long)(self->fd < 0)); } static PyObject * -get_closefd(fileio *self, void *closure) +fileio_get_closefd(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyBool_FromLong((long)(self->closefd)); } static PyObject * -get_mode(fileio *self, void *closure) +fileio_get_mode(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyUnicode_FromString(mode_string(self)); } static PyObject * -get_blksize(fileio *self, void *closure) +fileio_get_blksize(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) { return PyLong_FromLong(self->stat_atopen->st_blksize); @@ -1262,11 +1292,11 @@ get_blksize(fileio *self, void *closure) } static PyGetSetDef fileio_getsetlist[] = { - {"closed", (getter)get_closed, NULL, "True if the file is closed"}, - {"closefd", (getter)get_closefd, NULL, + {"closed", fileio_get_closed, NULL, "True if the file is closed"}, + {"closefd", fileio_get_closefd, NULL, "True if the file descriptor will be closed by close()."}, - {"mode", (getter)get_mode, NULL, "String giving the file mode"}, - {"_blksize", (getter)get_blksize, NULL, "Stat st_blksize if available"}, + {"mode", fileio_get_mode, NULL, "String giving the file mode"}, + {"_blksize", fileio_get_blksize, NULL, "Stat st_blksize if available"}, {NULL}, }; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 6d48bcb552b4bf..f558613dc6233c 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -353,7 +353,7 @@ _stringio_readline(stringio *self, Py_ssize_t limit) /* In case of overseek, return the empty string */ if (self->pos >= self->string_size) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); start = self->buf + self->pos; if (limit < 0 || limit > self->string_size - self->pos) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 439e26c5271939..68d16361962412 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -559,7 +559,7 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) Py_DECREF(state); } else { - buffer = PyBytes_FromString(""); + buffer = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); flag = 0; } flag <<= 1; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index ec5c298066a587..d7cb5abfdc0abd 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1128,6 +1128,7 @@ static PyMethodDef winconsoleio_methods[] = { _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF + {"_isatty_open_only", (PyCFunction)_io__WindowsConsoleIO_isatty, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_json.c b/Modules/_json.c index 9e29de0f22465f..ce0093ab431d05 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -86,11 +86,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); static PyObject * _encoded_const(PyObject *obj); static void @@ -1268,38 +1268,39 @@ encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { /* Python callable interface to encode_listencode_obj */ static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj, *result; + PyObject *obj; Py_ssize_t indent_level; - _PyUnicodeWriter writer; if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:_iterencode", kwlist, - &obj, &indent_level)) + &obj, &indent_level)) return NULL; - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } PyObject *newline_indent = NULL; if (self->indent != Py_None) { newline_indent = _create_newline_indent(self->indent, indent_level); if (newline_indent == NULL) { - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } } - if (encoder_listencode_obj(self, &writer, obj, newline_indent)) { - _PyUnicodeWriter_Dealloc(&writer); + if (encoder_listencode_obj(self, writer, obj, newline_indent)) { + PyUnicodeWriter_Discard(writer); Py_XDECREF(newline_indent); return NULL; } Py_XDECREF(newline_indent); - result = PyTuple_New(1); - if (result == NULL || - PyTuple_SetItem(result, 0, _PyUnicodeWriter_Finish(&writer)) < 0) { - Py_XDECREF(result); + PyObject *str = PyUnicodeWriter_Finish(writer); + if (str == NULL) { return NULL; } + PyObject *result = PyTuple_Pack(1, str); + Py_DECREF(str); return result; } @@ -1370,16 +1371,16 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj) } static int -_steal_accumulate(_PyUnicodeWriter *writer, PyObject *stolen) +_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen) { /* Append stolen and then decrement its reference count */ - int rval = _PyUnicodeWriter_WriteStr(writer, stolen); + int rval = PyUnicodeWriter_WriteStr(writer, stolen); Py_DECREF(stolen); return rval; } static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent) { /* Encode Python object obj to a JSON term */ @@ -1387,13 +1388,13 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, int rv; if (obj == Py_None) { - return _PyUnicodeWriter_WriteASCIIString(writer, "null", 4); + return PyUnicodeWriter_WriteUTF8(writer, "null", 4); } else if (obj == Py_True) { - return _PyUnicodeWriter_WriteASCIIString(writer, "true", 4); + return PyUnicodeWriter_WriteUTF8(writer, "true", 4); } else if (obj == Py_False) { - return _PyUnicodeWriter_WriteASCIIString(writer, "false", 5); + return PyUnicodeWriter_WriteUTF8(writer, "false", 5); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); @@ -1402,6 +1403,10 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, return _steal_accumulate(writer, encoded); } else if (PyLong_Check(obj)) { + if (PyLong_CheckExact(obj)) { + // Fast-path for exact integers + return PyUnicodeWriter_WriteRepr(writer, obj); + } PyObject *encoded = PyLong_Type.tp_repr(obj); if (encoded == NULL) return -1; @@ -1478,7 +1483,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, } static int -encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, +encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *first, PyObject *dct, PyObject *key, PyObject *value, PyObject *newline_indent, PyObject *item_separator) @@ -1518,7 +1523,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir *first = false; } else { - if (_PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { + if (PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { Py_DECREF(keystr); return -1; } @@ -1533,7 +1538,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir if (_steal_accumulate(writer, encoded) < 0) { return -1; } - if (_PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { + if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { @@ -1544,7 +1549,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir } static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent) { /* Encode Python dict dct a JSON term */ @@ -1555,8 +1560,10 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *new_newline_indent = NULL; PyObject *separator_indent = NULL; - if (PyDict_GET_SIZE(dct) == 0) /* Fast path */ - return _PyUnicodeWriter_WriteASCIIString(writer, "{}", 2); + if (PyDict_GET_SIZE(dct) == 0) { + /* Fast path */ + return PyUnicodeWriter_WriteUTF8(writer, "{}", 2); + } if (s->markers != Py_None) { int has_key; @@ -1574,8 +1581,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } } - if (_PyUnicodeWriter_WriteChar(writer, '{')) + if (PyUnicodeWriter_WriteChar(writer, '{')) { goto bail; + } PyObject *current_item_separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { @@ -1589,7 +1597,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } // update item separator with a borrowed reference current_item_separator = separator_indent; - if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + if (PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { goto bail; } } @@ -1635,13 +1643,14 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_CLEAR(new_newline_indent); Py_CLEAR(separator_indent); - if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + if (PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { goto bail; } } - if (_PyUnicodeWriter_WriteChar(writer, '}')) + if (PyUnicodeWriter_WriteChar(writer, '}')) { goto bail; + } return 0; bail: @@ -1653,7 +1662,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent) { PyObject *ident = NULL; @@ -1668,7 +1677,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); + return PyUnicodeWriter_WriteUTF8(writer, "[]", 2); } if (s->markers != Py_None) { @@ -1687,8 +1696,9 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, } } - if (_PyUnicodeWriter_WriteChar(writer, '[')) + if (PyUnicodeWriter_WriteChar(writer, '[')) { goto bail; + } PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { @@ -1697,7 +1707,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; } - if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + if (PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { goto bail; } @@ -1710,7 +1720,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) + if (PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) { @@ -1727,13 +1737,14 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (s->indent != Py_None) { Py_CLEAR(new_newline_indent); Py_CLEAR(separator_indent); - if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + if (PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { goto bail; } } - if (_PyUnicodeWriter_WriteChar(writer, ']')) + if (PyUnicodeWriter_WriteChar(writer, ']')) { goto bail; + } Py_DECREF(s_fast); return 0; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index de7395b610e133..0daec646605775 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -144,6 +144,17 @@ locale_is_ascii(const char *str) return (strlen(str) == 1 && ((unsigned char)str[0]) <= 127); } +static int +is_all_ascii(const char *str) +{ + for (; *str; str++) { + if ((unsigned char)*str > 127) { + return 0; + } + } + return 1; +} + static int locale_decode_monetary(PyObject *dict, struct lconv *lc) { @@ -478,113 +489,153 @@ _locale__getdefaultlocale_impl(PyObject *module) #endif #ifdef HAVE_LANGINFO_H -#define LANGINFO(X) {#X, X} +#define LANGINFO(X, Y) {#X, X, Y} static struct langinfo_constant{ - char* name; + const char *name; int value; + int category; } langinfo_constants[] = { /* These constants should exist on any langinfo implementation */ - LANGINFO(DAY_1), - LANGINFO(DAY_2), - LANGINFO(DAY_3), - LANGINFO(DAY_4), - LANGINFO(DAY_5), - LANGINFO(DAY_6), - LANGINFO(DAY_7), - - LANGINFO(ABDAY_1), - LANGINFO(ABDAY_2), - LANGINFO(ABDAY_3), - LANGINFO(ABDAY_4), - LANGINFO(ABDAY_5), - LANGINFO(ABDAY_6), - LANGINFO(ABDAY_7), - - LANGINFO(MON_1), - LANGINFO(MON_2), - LANGINFO(MON_3), - LANGINFO(MON_4), - LANGINFO(MON_5), - LANGINFO(MON_6), - LANGINFO(MON_7), - LANGINFO(MON_8), - LANGINFO(MON_9), - LANGINFO(MON_10), - LANGINFO(MON_11), - LANGINFO(MON_12), - - LANGINFO(ABMON_1), - LANGINFO(ABMON_2), - LANGINFO(ABMON_3), - LANGINFO(ABMON_4), - LANGINFO(ABMON_5), - LANGINFO(ABMON_6), - LANGINFO(ABMON_7), - LANGINFO(ABMON_8), - LANGINFO(ABMON_9), - LANGINFO(ABMON_10), - LANGINFO(ABMON_11), - LANGINFO(ABMON_12), + LANGINFO(DAY_1, LC_TIME), + LANGINFO(DAY_2, LC_TIME), + LANGINFO(DAY_3, LC_TIME), + LANGINFO(DAY_4, LC_TIME), + LANGINFO(DAY_5, LC_TIME), + LANGINFO(DAY_6, LC_TIME), + LANGINFO(DAY_7, LC_TIME), + + LANGINFO(ABDAY_1, LC_TIME), + LANGINFO(ABDAY_2, LC_TIME), + LANGINFO(ABDAY_3, LC_TIME), + LANGINFO(ABDAY_4, LC_TIME), + LANGINFO(ABDAY_5, LC_TIME), + LANGINFO(ABDAY_6, LC_TIME), + LANGINFO(ABDAY_7, LC_TIME), + + LANGINFO(MON_1, LC_TIME), + LANGINFO(MON_2, LC_TIME), + LANGINFO(MON_3, LC_TIME), + LANGINFO(MON_4, LC_TIME), + LANGINFO(MON_5, LC_TIME), + LANGINFO(MON_6, LC_TIME), + LANGINFO(MON_7, LC_TIME), + LANGINFO(MON_8, LC_TIME), + LANGINFO(MON_9, LC_TIME), + LANGINFO(MON_10, LC_TIME), + LANGINFO(MON_11, LC_TIME), + LANGINFO(MON_12, LC_TIME), + + LANGINFO(ABMON_1, LC_TIME), + LANGINFO(ABMON_2, LC_TIME), + LANGINFO(ABMON_3, LC_TIME), + LANGINFO(ABMON_4, LC_TIME), + LANGINFO(ABMON_5, LC_TIME), + LANGINFO(ABMON_6, LC_TIME), + LANGINFO(ABMON_7, LC_TIME), + LANGINFO(ABMON_8, LC_TIME), + LANGINFO(ABMON_9, LC_TIME), + LANGINFO(ABMON_10, LC_TIME), + LANGINFO(ABMON_11, LC_TIME), + LANGINFO(ABMON_12, LC_TIME), #ifdef RADIXCHAR /* The following are not available with glibc 2.0 */ - LANGINFO(RADIXCHAR), - LANGINFO(THOUSEP), + LANGINFO(RADIXCHAR, LC_NUMERIC), + LANGINFO(THOUSEP, LC_NUMERIC), /* YESSTR and NOSTR are deprecated in glibc, since they are a special case of message translation, which should be rather done using gettext. So we don't expose it to Python in the first place. - LANGINFO(YESSTR), - LANGINFO(NOSTR), + LANGINFO(YESSTR, LC_MESSAGES), + LANGINFO(NOSTR, LC_MESSAGES), */ - LANGINFO(CRNCYSTR), + LANGINFO(CRNCYSTR, LC_MONETARY), #endif - LANGINFO(D_T_FMT), - LANGINFO(D_FMT), - LANGINFO(T_FMT), - LANGINFO(AM_STR), - LANGINFO(PM_STR), + LANGINFO(D_T_FMT, LC_TIME), + LANGINFO(D_FMT, LC_TIME), + LANGINFO(T_FMT, LC_TIME), + LANGINFO(AM_STR, LC_TIME), + LANGINFO(PM_STR, LC_TIME), /* The following constants are available only with XPG4, but... OpenBSD doesn't have CODESET but has T_FMT_AMPM, and doesn't have a few of the others. Solution: ifdef-test them all. */ #ifdef CODESET - LANGINFO(CODESET), + LANGINFO(CODESET, LC_CTYPE), #endif #ifdef T_FMT_AMPM - LANGINFO(T_FMT_AMPM), + LANGINFO(T_FMT_AMPM, LC_TIME), #endif #ifdef ERA - LANGINFO(ERA), + LANGINFO(ERA, LC_TIME), #endif #ifdef ERA_D_FMT - LANGINFO(ERA_D_FMT), + LANGINFO(ERA_D_FMT, LC_TIME), #endif #ifdef ERA_D_T_FMT - LANGINFO(ERA_D_T_FMT), + LANGINFO(ERA_D_T_FMT, LC_TIME), #endif #ifdef ERA_T_FMT - LANGINFO(ERA_T_FMT), + LANGINFO(ERA_T_FMT, LC_TIME), #endif #ifdef ALT_DIGITS - LANGINFO(ALT_DIGITS), + LANGINFO(ALT_DIGITS, LC_TIME), #endif #ifdef YESEXPR - LANGINFO(YESEXPR), + LANGINFO(YESEXPR, LC_MESSAGES), #endif #ifdef NOEXPR - LANGINFO(NOEXPR), + LANGINFO(NOEXPR, LC_MESSAGES), #endif #ifdef _DATE_FMT /* This is not available in all glibc versions that have CODESET. */ - LANGINFO(_DATE_FMT), + LANGINFO(_DATE_FMT, LC_TIME), #endif - {0, 0} + {0, 0, 0} }; +/* Temporary make the LC_CTYPE locale to be the same as + * the locale of the specified category. */ +static int +change_locale(int category, char **oldloc) +{ + /* Keep a copy of the LC_CTYPE locale */ + *oldloc = setlocale(LC_CTYPE, NULL); + if (!*oldloc) { + PyErr_SetString(PyExc_RuntimeError, "faild to get LC_CTYPE locale"); + return -1; + } + *oldloc = _PyMem_Strdup(*oldloc); + if (!*oldloc) { + PyErr_NoMemory(); + return -1; + } + + /* Set a new locale if it is different. */ + char *loc = setlocale(category, NULL); + if (loc == NULL || strcmp(loc, *oldloc) == 0) { + PyMem_Free(*oldloc); + *oldloc = NULL; + return 0; + } + + setlocale(LC_CTYPE, loc); + return 1; +} + +/* Restore the old LC_CTYPE locale. */ +static void +restore_locale(char *oldloc) +{ + if (oldloc != NULL) { + setlocale(LC_CTYPE, oldloc); + PyMem_Free(oldloc); + } +} + /*[clinic input] _locale.nl_langinfo @@ -602,14 +653,50 @@ _locale_nl_langinfo_impl(PyObject *module, int item) /* Check whether this is a supported constant. GNU libc sometimes returns numeric values in the char* return value, which would crash PyUnicode_FromString. */ - for (i = 0; langinfo_constants[i].name; i++) + for (i = 0; langinfo_constants[i].name; i++) { if (langinfo_constants[i].value == item) { /* Check NULL as a workaround for GNU libc's returning NULL instead of an empty string for nl_langinfo(ERA). */ const char *result = nl_langinfo(item); result = result != NULL ? result : ""; - return PyUnicode_DecodeLocale(result, NULL); + char *oldloc = NULL; + if (langinfo_constants[i].category != LC_CTYPE + && !is_all_ascii(result) + && change_locale(langinfo_constants[i].category, &oldloc) < 0) + { + return NULL; + } + PyObject *pyresult; +#ifdef ALT_DIGITS + if (item == ALT_DIGITS) { + /* The result is a sequence of up to 100 NUL-separated strings. */ + const char *s = result; + int count = 0; + for (; count < 100 && *s; count++) { + s += strlen(s) + 1; + } + pyresult = PyTuple_New(count); + if (pyresult != NULL) { + for (int i = 0; i < count; i++) { + PyObject *unicode = PyUnicode_DecodeLocale(result, NULL); + if (unicode == NULL) { + Py_CLEAR(pyresult); + break; + } + PyTuple_SET_ITEM(pyresult, i, unicode); + result += strlen(result) + 1; + } + } + } + else +#endif + { + pyresult = PyUnicode_DecodeLocale(result, NULL); + } + restore_locale(oldloc); + return pyresult; } + } PyErr_SetString(PyExc_ValueError, "unsupported langinfo constant"); return NULL; } @@ -881,9 +968,9 @@ locale_clear(PyObject *module) } static void -locale_free(PyObject *module) +locale_free(void *module) { - locale_clear(module); + locale_clear((PyObject*)module); } static struct PyModuleDef _localemodule = { @@ -895,7 +982,7 @@ static struct PyModuleDef _localemodule = { _locale_slots, locale_traverse, locale_clear, - (freefunc)locale_free, + locale_free, }; PyMODINIT_FUNC diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 18affdd4875f3b..b2bd9545c1b130 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2146,7 +2146,7 @@ save_long(PicklerObject *self, PyObject *obj) if (self->proto >= 2) { /* Linear-time pickling. */ - uint64_t nbits; + int64_t nbits; size_t nbytes; unsigned char *pdata; char header[5]; @@ -2161,8 +2161,8 @@ save_long(PicklerObject *self, PyObject *obj) return 0; } nbits = _PyLong_NumBits(obj); - if (nbits == (uint64_t)-1 && PyErr_Occurred()) - goto error; + assert(nbits >= 0); + assert(!PyErr_Occurred()); /* How many bytes do we need? There are nbits >> 3 full * bytes of data, and nbits & 7 leftover bits. If there * are any leftover bits, then we clearly need another diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 3835a3072d96c6..ad66df47349db0 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -295,7 +295,7 @@ random_seed(RandomObject *self, PyObject *arg) int result = -1; /* guilty until proved innocent */ PyObject *n = NULL; uint32_t *key = NULL; - uint64_t bits; + int64_t bits; size_t keyused; int res; @@ -335,8 +335,8 @@ random_seed(RandomObject *self, PyObject *arg) /* Now split n into 32-bit chunks, from the right. */ bits = _PyLong_NumBits(n); - if (bits == (uint64_t)-1 && PyErr_Occurred()) - goto Done; + assert(bits >= 0); + assert(!PyErr_Occurred()); /* Figure out how many 32-bit chunks this gives us. */ keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 1f5f0215980971..f2d3b331226a7a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2251,6 +2251,17 @@ PySSL_dealloc(PySSLSocket *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); if (self->ssl) { + // If we free the SSL socket object without having called SSL_shutdown, + // OpenSSL will invalidate the linked SSL session object. While this + // behavior is strictly RFC-compliant, it makes session reuse less + // likely and it would also break compatibility with older stdlib + // versions (which used an ugly workaround of duplicating the + // SSL_SESSION object). + // Therefore, we ensure the socket is marked as shutdown in any case. + // + // See elaborate explanation at + // https://github.com/python/cpython/pull/123249#discussion_r1766164530 + SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } Py_XDECREF(self->Socket); @@ -2795,48 +2806,6 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) #endif } -static SSL_SESSION* -_ssl_session_dup(SSL_SESSION *session) { - SSL_SESSION *newsession = NULL; - int slen; - unsigned char *senc = NULL, *p; - const unsigned char *const_p; - - if (session == NULL) { - PyErr_SetString(PyExc_ValueError, "Invalid session"); - goto error; - } - - /* get length */ - slen = i2d_SSL_SESSION(session, NULL); - if (slen == 0 || slen > 0xFF00) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - if ((senc = PyMem_Malloc(slen)) == NULL) { - PyErr_NoMemory(); - goto error; - } - p = senc; - if (!i2d_SSL_SESSION(session, &p)) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - const_p = senc; - newsession = d2i_SSL_SESSION(NULL, &const_p, slen); - if (newsession == NULL) { - PyErr_SetString(PyExc_ValueError, "d2i() failed"); - goto error; - } - PyMem_Free(senc); - return newsession; - error: - if (senc != NULL) { - PyMem_Free(senc); - } - return NULL; -} - static PyObject * PySSL_get_session(PySSLSocket *self, void *closure) { /* get_session can return sessions from a server-side connection, @@ -2844,15 +2813,6 @@ PySSL_get_session(PySSLSocket *self, void *closure) { PySSLSession *pysess; SSL_SESSION *session; - /* duplicate session as workaround for session bug in OpenSSL 1.1.0, - * https://github.com/openssl/openssl/issues/1550 */ - session = SSL_get0_session(self->ssl); /* borrowed reference */ - if (session == NULL) { - Py_RETURN_NONE; - } - if ((session = _ssl_session_dup(session)) == NULL) { - return NULL; - } session = SSL_get1_session(self->ssl); if (session == NULL) { Py_RETURN_NONE; @@ -2871,11 +2831,8 @@ PySSL_get_session(PySSLSocket *self, void *closure) { } static int PySSL_set_session(PySSLSocket *self, PyObject *value, - void *closure) - { + void *closure) { PySSLSession *pysess; - SSL_SESSION *session; - int result; if (!Py_IS_TYPE(value, get_state_sock(self)->PySSLSession_Type)) { PyErr_SetString(PyExc_TypeError, "Value is not a SSLSession."); @@ -2898,14 +2855,7 @@ static int PySSL_set_session(PySSLSocket *self, PyObject *value, "Cannot set session after handshake."); return -1; } - /* duplicate session */ - if ((session = _ssl_session_dup(pysess->session)) == NULL) { - return -1; - } - result = SSL_set_session(self->ssl, session); - /* free duplicate, SSL_set_session() bumps ref count */ - SSL_SESSION_free(session); - if (result == 0) { + if (SSL_set_session(self->ssl, pysess->session) == 0) { _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); return -1; } diff --git a/Modules/_struct.c b/Modules/_struct.c index 2ae5060ba34163..21582b945be23d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -12,6 +12,9 @@ #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() +#ifdef Py_HAVE_C_COMPLEX +# include "_complex.h" // complex +#endif #include // offsetof() /*[clinic input] @@ -80,6 +83,10 @@ typedef struct { char c; int x; } st_int; typedef struct { char c; long x; } st_long; typedef struct { char c; float x; } st_float; typedef struct { char c; double x; } st_double; +#ifdef Py_HAVE_C_COMPLEX +typedef struct { char c; float complex x; } st_float_complex; +typedef struct { char c; double complex x; } st_double_complex; +#endif typedef struct { char c; void *x; } st_void_p; typedef struct { char c; size_t x; } st_size_t; typedef struct { char c; _Bool x; } st_bool; @@ -89,6 +96,10 @@ typedef struct { char c; _Bool x; } st_bool; #define LONG_ALIGN (sizeof(st_long) - sizeof(long)) #define FLOAT_ALIGN (sizeof(st_float) - sizeof(float)) #define DOUBLE_ALIGN (sizeof(st_double) - sizeof(double)) +#ifdef Py_HAVE_C_COMPLEX +# define FLOAT_COMPLEX_ALIGN (sizeof(st_float_complex) - sizeof(float complex)) +# define DOUBLE_COMPLEX_ALIGN (sizeof(st_double_complex) - sizeof(double complex)) +#endif #define VOID_P_ALIGN (sizeof(st_void_p) - sizeof(void *)) #define SIZE_T_ALIGN (sizeof(st_size_t) - sizeof(size_t)) #define BOOL_ALIGN (sizeof(st_bool) - sizeof(_Bool)) @@ -407,7 +418,7 @@ static PyObject * nu_short(_structmodulestate *state, const char *p, const formatdef *f) { short x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -415,7 +426,7 @@ static PyObject * nu_ushort(_structmodulestate *state, const char *p, const formatdef *f) { unsigned short x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -423,7 +434,7 @@ static PyObject * nu_int(_structmodulestate *state, const char *p, const formatdef *f) { int x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -431,7 +442,7 @@ static PyObject * nu_uint(_structmodulestate *state, const char *p, const formatdef *f) { unsigned int x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLong((unsigned long)x); } @@ -439,7 +450,7 @@ static PyObject * nu_long(_structmodulestate *state, const char *p, const formatdef *f) { long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong(x); } @@ -447,7 +458,7 @@ static PyObject * nu_ulong(_structmodulestate *state, const char *p, const formatdef *f) { unsigned long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLong(x); } @@ -455,7 +466,7 @@ static PyObject * nu_ssize_t(_structmodulestate *state, const char *p, const formatdef *f) { Py_ssize_t x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromSsize_t(x); } @@ -463,7 +474,7 @@ static PyObject * nu_size_t(_structmodulestate *state, const char *p, const formatdef *f) { size_t x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromSize_t(x); } @@ -471,7 +482,7 @@ static PyObject * nu_longlong(_structmodulestate *state, const char *p, const formatdef *f) { long long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLongLong(x); } @@ -479,16 +490,15 @@ static PyObject * nu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f) { unsigned long long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLongLong(x); } static PyObject * nu_bool(_structmodulestate *state, const char *p, const formatdef *f) { - _Bool x; - memcpy((char *)&x, p, sizeof x); - return PyBool_FromLong(x != 0); + const _Bool bool_false = 0; + return PyBool_FromLong(memcmp(p, &bool_false, sizeof(_Bool))); } @@ -506,7 +516,7 @@ static PyObject * nu_float(_structmodulestate *state, const char *p, const formatdef *f) { float x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyFloat_FromDouble((double)x); } @@ -514,15 +524,35 @@ static PyObject * nu_double(_structmodulestate *state, const char *p, const formatdef *f) { double x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyFloat_FromDouble(x); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +nu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + float complex x; + + memcpy(&x, p, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} + +static PyObject * +nu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double complex x; + + memcpy(&x, p, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} +#endif + static PyObject * nu_void_p(_structmodulestate *state, const char *p, const formatdef *f) { void *x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromVoidPtr(x); } @@ -587,7 +617,7 @@ np_short(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 0); } y = (short)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -606,7 +636,7 @@ np_ushort(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 1); } y = (unsigned short)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -626,7 +656,7 @@ np_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 0); #endif y = (int)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -646,7 +676,7 @@ np_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) if (x > ((unsigned long)UINT_MAX)) RANGE_ERROR(state, f, 1); #endif - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -660,7 +690,7 @@ np_long(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -674,7 +704,7 @@ np_ulong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -688,7 +718,7 @@ np_ssize_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -702,7 +732,7 @@ np_size_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -720,7 +750,7 @@ np_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -737,7 +767,7 @@ np_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -751,7 +781,7 @@ np_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) if (y < 0) return -1; x = y; - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -774,7 +804,7 @@ np_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) "required argument is not a float"); return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -787,9 +817,61 @@ np_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) "required argument is not a float"); return -1; } - memcpy(p, (char *)&x, sizeof(double)); + memcpy(p, &x, sizeof(double)); + return 0; +} + +#ifdef Py_HAVE_C_COMPLEX +static int +np_float_complex(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + Py_complex c = PyComplex_AsCComplex(v); + float complex x = CMPLXF((float)c.real, (float)c.imag); + + if (c.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + memcpy(p, &x, sizeof(x)); + return 0; +} + +static int +np_double_complex(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + Py_complex c = PyComplex_AsCComplex(v); + double complex x = CMPLX(c.real, c.imag); + + if (c.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + memcpy(p, &x, sizeof(x)); return 0; } +#else +static int +np_complex_stub(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + PyErr_Format(state->StructError, + "'%c' format not supported on this system", + f->format); + return -1; +} +static PyObject * +nu_complex_stub(_structmodulestate *state, const char *p, const formatdef *f) +{ + PyErr_Format(state->StructError, + "'%c' format not supported on this system", + f->format); + return NULL; +} +#endif static int np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) @@ -804,7 +886,7 @@ np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) Py_DECREF(v); if (x == NULL && PyErr_Occurred()) return -1; - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -829,6 +911,13 @@ static const formatdef native_table[] = { {'e', sizeof(short), SHORT_ALIGN, nu_halffloat, np_halffloat}, {'f', sizeof(float), FLOAT_ALIGN, nu_float, np_float}, {'d', sizeof(double), DOUBLE_ALIGN, nu_double, np_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', sizeof(float complex), FLOAT_COMPLEX_ALIGN, nu_float_complex, np_float_complex}, + {'C', sizeof(double complex), DOUBLE_COMPLEX_ALIGN, nu_double_complex, np_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {'P', sizeof(void *), VOID_P_ALIGN, nu_void_p, np_void_p}, {0} }; @@ -929,6 +1018,38 @@ bu_double(_structmodulestate *state, const char *p, const formatdef *f) return unpack_double(p, 0); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +bu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x = PyFloat_Unpack4(p, 0); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + double y = PyFloat_Unpack4(p + 4, 0); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} + +static PyObject * +bu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x, y; + + x = PyFloat_Unpack8(p, 0); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + y = PyFloat_Unpack8(p + 8, 0); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} +#endif + static PyObject * bu_bool(_structmodulestate *state, const char *p, const formatdef *f) { @@ -1068,6 +1189,38 @@ bp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) return PyFloat_Pack8(x, p, 0); } +#ifdef Py_HAVE_C_COMPLEX +static int +bp_float_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack4(x.real, p, 0)) { + return -1; + } + return PyFloat_Pack4(x.imag, p + 4, 0); +} + +static int +bp_double_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack8(x.real, p, 0)) { + return -1; + } + return PyFloat_Pack8(x.imag, p + 8, 0); +} +#endif + static int bp_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -1098,6 +1251,13 @@ static formatdef bigendian_table[] = { {'e', 2, 0, bu_halffloat, bp_halffloat}, {'f', 4, 0, bu_float, bp_float}, {'d', 8, 0, bu_double, bp_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', 8, 0, bu_float_complex, bp_float_complex}, + {'C', 16, 0, bu_double_complex, bp_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {0} }; @@ -1197,6 +1357,38 @@ lu_double(_structmodulestate *state, const char *p, const formatdef *f) return unpack_double(p, 1); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +lu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x = PyFloat_Unpack4(p, 1); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + double y = PyFloat_Unpack4(p + 4, 1); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} + +static PyObject * +lu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x, y; + + x = PyFloat_Unpack8(p, 1); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + y = PyFloat_Unpack8(p + 8, 1); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} +#endif + static int lp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -1330,6 +1522,39 @@ lp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) return PyFloat_Pack8(x, p, 1); } +#ifdef Py_HAVE_C_COMPLEX +static int +lp_float_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack4(x.real, p, 1)) { + return -1; + } + return PyFloat_Pack4(x.imag, p + 4, 1); + +} + +static int +lp_double_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack8(x.real, p, 1)) { + return -1; + } + return PyFloat_Pack8(x.imag, p + 8, 1); +} +#endif + static formatdef lilendian_table[] = { {'x', 1, 0, NULL}, {'b', 1, 0, nu_byte, np_byte}, @@ -1350,6 +1575,13 @@ static formatdef lilendian_table[] = { {'e', 2, 0, lu_halffloat, lp_halffloat}, {'f', 4, 0, lu_float, lp_float}, {'d', 8, 0, lu_double, lp_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', 8, 0, lu_float_complex, lp_float_complex}, + {'C', 16, 0, lu_double_complex, lp_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {0} }; diff --git a/Modules/_testcapi/codec.c b/Modules/_testcapi/codec.c index d13f51e20331a1..e27e64e066c458 100644 --- a/Modules/_testcapi/codec.c +++ b/Modules/_testcapi/codec.c @@ -1,15 +1,219 @@ #include "parts.h" #include "util.h" +// === Codecs registration and un-registration ================================ + +static PyObject * +codec_register(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Register(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_unregister(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Unregister(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_known_encoding(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_KnownEncoding(encoding) ? Py_True : Py_False; +} + +// === Codecs encoding and decoding interfaces ================================ + +static PyObject * +codec_encode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Encode(input, encoding, errors); +} + +static PyObject * +codec_decode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Decode(input, encoding, errors); +} + +static PyObject * +codec_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Encoder(encoding); +} + +static PyObject * +codec_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Decoder(encoding); +} + +static PyObject * +codec_incremental_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incremental_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +codec_stream_reader(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamReader(encoding, stream, errors); +} + +static PyObject * +codec_stream_writer(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamWriter(encoding, stream, errors); +} + +// === Codecs errors handlers ================================================= + +static PyObject * +codec_register_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *encoding; // must not be NULL + PyObject *error; + if (!PyArg_ParseTuple(args, "sO", &encoding, &error)) { + return NULL; + } + if (PyCodec_RegisterError(encoding, error) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_lookup_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_LookupError(encoding); +} + +static PyObject * +codec_strict_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_StrictErrors(exc); +} + +static PyObject * +codec_ignore_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_IgnoreErrors(exc); +} + +static PyObject * +codec_replace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_ReplaceErrors(exc); +} + +static PyObject * +codec_xmlcharrefreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_XMLCharRefReplaceErrors(exc); +} + +static PyObject * +codec_backslashreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_BackslashReplaceErrors(exc); +} static PyMethodDef test_methods[] = { - {NULL}, + /* codecs registration */ + {"codec_register", codec_register, METH_O}, + {"codec_unregister", codec_unregister, METH_O}, + {"codec_known_encoding", codec_known_encoding, METH_VARARGS}, + /* encoding and decoding interface */ + {"codec_encode", codec_encode, METH_VARARGS}, + {"codec_decode", codec_decode, METH_VARARGS}, + {"codec_encoder", codec_encoder, METH_VARARGS}, + {"codec_decoder", codec_decoder, METH_VARARGS}, + {"codec_incremental_encoder", codec_incremental_encoder, METH_VARARGS}, + {"codec_incremental_decoder", codec_incremental_decoder, METH_VARARGS}, + {"codec_stream_reader", codec_stream_reader, METH_VARARGS}, + {"codec_stream_writer", codec_stream_writer, METH_VARARGS}, + /* error handling */ + {"codec_register_error", codec_register_error, METH_VARARGS}, + {"codec_lookup_error", codec_lookup_error, METH_VARARGS}, + {"codec_strict_errors", codec_strict_errors, METH_O}, + {"codec_ignore_errors", codec_ignore_errors, METH_O}, + {"codec_replace_errors", codec_replace_errors, METH_O}, + {"codec_xmlcharrefreplace_errors", codec_xmlcharrefreplace_errors, METH_O}, + {"codec_backslashreplace_errors", codec_backslashreplace_errors, METH_O}, + // PyCodec_NameReplaceErrors() is tested in _testlimitedcapi/codec.c + {NULL, NULL, 0, NULL}, }; int _PyTestCapi_Init_Codec(PyObject *m) { - if (PyModule_AddFunctions(m, test_methods) < 0){ + if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; } diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index f3d54215e04232..b800f9b8eb3473 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -129,7 +129,7 @@ static PyObject * get_timezones_offset_zero(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, 0, 0); - PyObject *name = PyUnicode_FromString(""); + PyObject *name = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (offset == NULL || name == NULL) { Py_XDECREF(offset); Py_XDECREF(name); diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index e80d898118daa5..307797f98f12ae 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -181,19 +181,6 @@ dict_popstring_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_PopString(dict, key, NULL)); } -static PyObject * -dict_version(PyObject *self, PyObject *dict) -{ - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, "expected dict"); - return NULL; - } -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag); -_Py_COMP_DIAG_POP -} - static PyMethodDef test_methods[] = { {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_getitemref", dict_getitemref, METH_VARARGS}, @@ -204,7 +191,6 @@ static PyMethodDef test_methods[] = { {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, - {"dict_version", dict_version, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/util.h b/Modules/_testcapi/util.h index f26d7656a10138..042e522542eddb 100644 --- a/Modules/_testcapi/util.h +++ b/Modules/_testcapi/util.h @@ -31,3 +31,13 @@ static const char uninitialized[] = "uninitialized"; #define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) /* Marker to check that integer value was set. */ #define UNINITIALIZED_INT (63256717) +/* + * Marker to indicate that a NULL parameter would not be allowed + * at runtime but that the test interface will check that it is + * indeed the case. + * + * Use this macro only if passing NULL to the C API would raise + * a catchable exception (and not a fatal exception that would + * crash the interpreter). + */ + #define NULL_WOULD_RAISE(NAME) NAME diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 689863d098ad8a..b4233d07134aea 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -630,7 +630,7 @@ static int num_context_object_enter_events[NUM_CONTEXT_WATCHERS] = {0, 0}; static int num_context_object_exit_events[NUM_CONTEXT_WATCHERS] = {0, 0}; static int -handle_context_watcher_event(int which_watcher, PyContextEvent event, PyContext *ctx) { +handle_context_watcher_event(int which_watcher, PyContextEvent event, PyObject *ctx) { if (event == Py_CONTEXT_EVENT_ENTER) { num_context_object_enter_events[which_watcher]++; } @@ -644,22 +644,22 @@ handle_context_watcher_event(int which_watcher, PyContextEvent event, PyContext } static int -first_context_watcher_callback(PyContextEvent event, PyContext *ctx) { +first_context_watcher_callback(PyContextEvent event, PyObject *ctx) { return handle_context_watcher_event(0, event, ctx); } static int -second_context_watcher_callback(PyContextEvent event, PyContext *ctx) { +second_context_watcher_callback(PyContextEvent event, PyObject *ctx) { return handle_context_watcher_event(1, event, ctx); } static int -noop_context_event_handler(PyContextEvent event, PyContext *ctx) { +noop_context_event_handler(PyContextEvent event, PyObject *ctx) { return 0; } static int -error_context_event_handler(PyContextEvent event, PyContext *ctx) { +error_context_event_handler(PyContextEvent event, PyObject *ctx) { PyErr_SetString(PyExc_RuntimeError, "boom!"); return -1; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5966eb674cf4e5..ea26295cca49d4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1909,25 +1909,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -dict_get_version(PyObject *self, PyObject *args) -{ - PyDictObject *dict; - uint64_t version; - - if (!PyArg_ParseTuple(args, "O!", &PyDict_Type, &dict)) - return NULL; - - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS - version = dict->ma_version_tag; - _Py_COMP_DIAG_POP - - static_assert(sizeof(unsigned long long) >= sizeof(version), - "version is larger than unsigned long long"); - return PyLong_FromUnsignedLongLong((unsigned long long)version); -} - static PyObject * raise_SIGINT_then_send_None(PyObject *self, PyObject *args) @@ -3329,6 +3310,35 @@ test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +// Used by `finalize_thread_hang`. +#ifdef _POSIX_THREADS +static void finalize_thread_hang_cleanup_callback(void *Py_UNUSED(arg)) { + // Should not reach here. + Py_FatalError("pthread thread termination was triggered unexpectedly"); +} +#endif + +// Tests that finalization does not trigger pthread cleanup. +// +// Must be called with a single nullary callable function that should block +// (with GIL released) until finalization is in progress. +static PyObject * +finalize_thread_hang(PyObject *self, PyObject *callback) +{ + // WASI builds some pthread stuff but doesn't have these APIs today? +#if defined(_POSIX_THREADS) && !defined(__wasi__) + pthread_cleanup_push(finalize_thread_hang_cleanup_callback, NULL); +#endif + PyObject_CallNoArgs(callback); + // Should not reach here. + Py_FatalError("thread unexpectedly did not hang"); +#if defined(_POSIX_THREADS) && !defined(__wasi__) + pthread_cleanup_pop(0); +#endif + Py_RETURN_NONE; +} + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3407,7 +3417,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, #ifdef W_STOPCODE @@ -3469,6 +3478,7 @@ static PyMethodDef TestMethods[] = { {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, {"function_set_warning", function_set_warning, METH_NOARGS}, {"test_critical_sections", test_critical_sections, METH_NOARGS}, + {"finalize_thread_hang", finalize_thread_hang, METH_O, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 0451688a46c75f..c403075fbb2501 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -681,13 +681,13 @@ set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) { - if (PyFunction_Check(f->f_funcobj)) { + if (PyStackRef_FunctionCheck(f->f_funcobj)) { + PyFunctionObject *func = _PyFrame_GetFunction(f); PyObject *module = _get_current_module(); assert(module != NULL); module_state *state = get_module_state(module); Py_DECREF(module); - int res = PyList_Append(state->record_list, - ((PyFunctionObject *)f->f_funcobj)->func_name); + int res = PyList_Append(state->record_list, func->func_name); if (res < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index e74cbfe19871bf..ba83a23117b2a5 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -38,6 +38,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Codec(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Complex(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/codec.c b/Modules/_testlimitedcapi/codec.c new file mode 100644 index 00000000000000..fdc18eedc2d288 --- /dev/null +++ b/Modules/_testlimitedcapi/codec.c @@ -0,0 +1,29 @@ +#include "pyconfig.h" // Py_GIL_DISABLED + +// Need limited C API version 3.5 for PyCodec_NameReplaceErrors() +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" + +static PyObject * +codec_namereplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_NameReplaceErrors(exc); +} + +static PyMethodDef test_methods[] = { + {"codec_namereplace_errors", codec_namereplace_errors, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Codec(PyObject *module) +{ + if (PyModule_AddFunctions(module, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 12b890853803f4..4107b150c5b4e0 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,6 +25,7 @@ int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Codec(PyObject *module); int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); int _PyTestLimitedCAPI_Init_Eval(PyObject *module); diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c index 2b70d09108a333..c7a23d5d1cbd71 100644 --- a/Modules/_testlimitedcapi/unicode.c +++ b/Modules/_testlimitedcapi/unicode.c @@ -1,7 +1,7 @@ #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED - // Need limited C API 3.13 to test PyUnicode_EqualToUTF8() -# define Py_LIMITED_API 0x030d0000 + // Need limited C API 3.14 to test PyUnicode_Equal() +# define Py_LIMITED_API 0x030e0000 #endif #include "parts.h" @@ -1837,6 +1837,23 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) #undef CHECK_FORMAT_0 } + +/* Test PyUnicode_Equal() */ +static PyObject * +unicode_equal(PyObject *module, PyObject *args) +{ + PyObject *str1, *str2; + if (!PyArg_ParseTuple(args, "OO", &str1, &str2)) { + return NULL; + } + + NULLABLE(str1); + NULLABLE(str2); + RETURN_INT(PyUnicode_Equal(str1, str2)); +} + + + static PyMethodDef TestMethods[] = { {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, @@ -1924,6 +1941,7 @@ static PyMethodDef TestMethods[] = { {"unicode_format", unicode_format, METH_VARARGS}, {"unicode_contains", unicode_contains, METH_VARARGS}, {"unicode_isidentifier", unicode_isidentifier, METH_O}, + {"unicode_equal", unicode_equal, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index b3ed8e7bc56b9e..d4408aa9e42d9d 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -599,8 +599,9 @@ PyThreadHandleObject_traverse(PyThreadHandleObject *self, visitproc visit, } static void -PyThreadHandleObject_dealloc(PyThreadHandleObject *self) +PyThreadHandleObject_dealloc(PyObject *op) { + PyThreadHandleObject *self = (PyThreadHandleObject*)op; PyObject_GC_UnTrack(self); PyTypeObject *tp = Py_TYPE(self); ThreadHandle_decref(self->handle); @@ -609,23 +610,26 @@ PyThreadHandleObject_dealloc(PyThreadHandleObject *self) } static PyObject * -PyThreadHandleObject_repr(PyThreadHandleObject *self) +PyThreadHandleObject_repr(PyObject *op) { + PyThreadHandleObject *self = (PyThreadHandleObject*)op; PyThread_ident_t ident = ThreadHandle_ident(self->handle); return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", Py_TYPE(self)->tp_name, ident); } static PyObject * -PyThreadHandleObject_get_ident(PyThreadHandleObject *self, - PyObject *Py_UNUSED(ignored)) +PyThreadHandleObject_get_ident(PyObject *op, void *Py_UNUSED(ignored)) { + PyThreadHandleObject *self = (PyThreadHandleObject*)op; return PyLong_FromUnsignedLongLong(ThreadHandle_ident(self->handle)); } static PyObject * -PyThreadHandleObject_join(PyThreadHandleObject *self, PyObject *args) +PyThreadHandleObject_join(PyObject *op, PyObject *args) { + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + PyObject *timeout_obj = NULL; if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; @@ -646,9 +650,9 @@ PyThreadHandleObject_join(PyThreadHandleObject *self, PyObject *args) } static PyObject * -PyThreadHandleObject_is_done(PyThreadHandleObject *self, - PyObject *Py_UNUSED(ignored)) +PyThreadHandleObject_is_done(PyObject *op, PyObject *Py_UNUSED(ignored)) { + PyThreadHandleObject *self = (PyThreadHandleObject*)op; if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { Py_RETURN_TRUE; } @@ -658,9 +662,9 @@ PyThreadHandleObject_is_done(PyThreadHandleObject *self, } static PyObject * -PyThreadHandleObject_set_done(PyThreadHandleObject *self, - PyObject *Py_UNUSED(ignored)) +PyThreadHandleObject_set_done(PyObject *op, PyObject *Py_UNUSED(ignored)) { + PyThreadHandleObject *self = (PyThreadHandleObject*)op; if (ThreadHandle_set_done(self->handle) < 0) { return NULL; } @@ -668,20 +672,20 @@ PyThreadHandleObject_set_done(PyThreadHandleObject *self, } static PyGetSetDef ThreadHandle_getsetlist[] = { - {"ident", (getter)PyThreadHandleObject_get_ident, NULL, NULL}, + {"ident", PyThreadHandleObject_get_ident, NULL, NULL}, {0}, }; static PyMethodDef ThreadHandle_methods[] = { - {"join", (PyCFunction)PyThreadHandleObject_join, METH_VARARGS, NULL}, - {"_set_done", (PyCFunction)PyThreadHandleObject_set_done, METH_NOARGS, NULL}, - {"is_done", (PyCFunction)PyThreadHandleObject_is_done, METH_NOARGS, NULL}, + {"join", PyThreadHandleObject_join, METH_VARARGS, NULL}, + {"_set_done", PyThreadHandleObject_set_done, METH_NOARGS, NULL}, + {"is_done", PyThreadHandleObject_is_done, METH_NOARGS, NULL}, {0, 0} }; static PyType_Slot ThreadHandle_Type_slots[] = { - {Py_tp_dealloc, (destructor)PyThreadHandleObject_dealloc}, - {Py_tp_repr, (reprfunc)PyThreadHandleObject_repr}, + {Py_tp_dealloc, PyThreadHandleObject_dealloc}, + {Py_tp_repr, PyThreadHandleObject_repr}, {Py_tp_getset, ThreadHandle_getsetlist}, {Py_tp_traverse, PyThreadHandleObject_traverse}, {Py_tp_methods, ThreadHandle_methods}, @@ -701,41 +705,27 @@ static PyType_Spec ThreadHandle_Type_spec = { typedef struct { PyObject_HEAD - PyThread_type_lock lock_lock; - PyObject *in_weakreflist; - char locked; /* for sanity checking */ + PyMutex lock; } lockobject; static int -lock_traverse(lockobject *self, visitproc visit, void *arg) +lock_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; } static void -lock_dealloc(lockobject *self) +lock_dealloc(PyObject *op) { + lockobject *self = (lockobject*)op; PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); - } - if (self->lock_lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->locked) - PyThread_release_lock(self->lock_lock); - PyThread_free_lock(self->lock_lock); - } + PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject*)self); Py_DECREF(tp); } -static inline PyLockStatus -acquire_timed(PyThread_type_lock lock, PyTime_t timeout) -{ - return PyThread_acquire_lock_timed_with_retries(lock, timeout); -} static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, @@ -784,19 +774,21 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, } static PyObject * -lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) +lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds) { + lockobject *self = (lockobject*)op; + PyTime_t timeout; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { return NULL; + } - PyLockStatus r = acquire_timed(self->lock_lock, timeout); + PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout, + _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { return NULL; } - if (r == PY_LOCK_ACQUIRED) - self->locked = 1; return PyBool_FromLong(r == PY_LOCK_ACQUIRED); } @@ -824,16 +816,15 @@ PyDoc_STRVAR(enter_doc, Lock the lock."); static PyObject * -lock_PyThread_release_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +lock_PyThread_release_lock(PyObject *op, PyObject *Py_UNUSED(ignored)) { + lockobject *self = (lockobject*)op; /* Sanity check: the lock must be locked */ - if (!self->locked) { + if (_PyMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(ThreadError, "release unlocked lock"); return NULL; } - self->locked = 0; - PyThread_release_lock(self->lock_lock); Py_RETURN_NONE; } @@ -858,9 +849,10 @@ PyDoc_STRVAR(lock_exit_doc, Release the lock."); static PyObject * -lock_locked_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +lock_locked_lock(PyObject *op, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong((long)self->locked); + lockobject *self = (lockobject*)op; + return PyBool_FromLong(PyMutex_IsLocked(&self->lock)); } PyDoc_STRVAR(locked_doc, @@ -876,23 +868,19 @@ PyDoc_STRVAR(locked_lock_doc, An obsolete synonym of locked()."); static PyObject * -lock_repr(lockobject *self) +lock_repr(PyObject *op) { + lockobject *self = (lockobject*)op; return PyUnicode_FromFormat("<%s %s object at %p>", - self->locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); + PyMutex_IsLocked(&self->lock) ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); } #ifdef HAVE_FORK static PyObject * -lock__at_fork_reinit(lockobject *self, PyObject *Py_UNUSED(args)) +lock__at_fork_reinit(PyObject *op, PyObject *Py_UNUSED(args)) { - if (_PyThread_at_fork_reinit(&self->lock_lock) < 0) { - PyErr_SetString(ThreadError, "failed to reinitialize lock at fork"); - return NULL; - } - - self->locked = 0; - + lockobject *self = (lockobject *)op; + _PyMutex_at_fork_reinit(&self->lock); Py_RETURN_NONE; } #endif /* HAVE_FORK */ @@ -924,20 +912,20 @@ static PyMethodDef lock_methods[] = { METH_VARARGS | METH_KEYWORDS, acquire_lock_doc}, {"acquire", _PyCFunction_CAST(lock_PyThread_acquire_lock), METH_VARARGS | METH_KEYWORDS, acquire_doc}, - {"release_lock", (PyCFunction)lock_PyThread_release_lock, + {"release_lock", lock_PyThread_release_lock, METH_NOARGS, release_lock_doc}, - {"release", (PyCFunction)lock_PyThread_release_lock, + {"release", lock_PyThread_release_lock, METH_NOARGS, release_doc}, - {"locked_lock", (PyCFunction)lock_locked_lock, + {"locked_lock", lock_locked_lock, METH_NOARGS, locked_lock_doc}, - {"locked", (PyCFunction)lock_locked_lock, + {"locked", lock_locked_lock, METH_NOARGS, locked_doc}, {"__enter__", _PyCFunction_CAST(lock_PyThread_acquire_lock), METH_VARARGS | METH_KEYWORDS, enter_doc}, - {"__exit__", (PyCFunction)lock_PyThread_release_lock, + {"__exit__", lock_PyThread_release_lock, METH_VARARGS, lock_exit_doc}, #ifdef HAVE_FORK - {"_at_fork_reinit", (PyCFunction)lock__at_fork_reinit, + {"_at_fork_reinit", lock__at_fork_reinit, METH_NOARGS, NULL}, #endif {NULL, NULL} /* sentinel */ @@ -958,18 +946,12 @@ A lock is not owned by the thread that locked it; another thread may\n\ unlock it. A thread attempting to lock a lock that it has already locked\n\ will block until another thread unlocks it. Deadlocks may ensue."); -static PyMemberDef lock_type_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, in_weakreflist), Py_READONLY}, - {NULL}, -}; - static PyType_Slot lock_type_slots[] = { - {Py_tp_dealloc, (destructor)lock_dealloc}, - {Py_tp_repr, (reprfunc)lock_repr}, + {Py_tp_dealloc, lock_dealloc}, + {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, - {Py_tp_members, lock_type_members}, {Py_tp_new, lock_new}, {0, 0} }; @@ -978,7 +960,7 @@ static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -986,10 +968,7 @@ static PyType_Spec lock_type_spec = { typedef struct { PyObject_HEAD - PyThread_type_lock rlock_lock; - PyThread_ident_t rlock_owner; - unsigned long rlock_count; - PyObject *in_weakreflist; + _PyRecursiveMutex lock; } rlockobject; static int @@ -1001,61 +980,30 @@ rlock_traverse(rlockobject *self, visitproc visit, void *arg) static void -rlock_dealloc(rlockobject *self) +rlock_dealloc(PyObject *op) { + rlockobject *self = (rlockobject*)op; PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - /* self->rlock_lock can be NULL if PyThread_allocate_lock() failed - in rlock_new() */ - if (self->rlock_lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->rlock_count > 0) - PyThread_release_lock(self->rlock_lock); - - PyThread_free_lock(self->rlock_lock); - } + PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); } -static bool -rlock_is_owned_by(rlockobject *self, PyThread_ident_t tid) -{ - PyThread_ident_t owner_tid = - _Py_atomic_load_ullong_relaxed(&self->rlock_owner); - return owner_tid == tid && self->rlock_count > 0; -} static PyObject * -rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) +rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds) { + rlockobject *self = (rlockobject*)op; PyTime_t timeout; - PyThread_ident_t tid; - PyLockStatus r = PY_LOCK_ACQUIRED; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { return NULL; - - tid = PyThread_get_thread_ident_ex(); - if (rlock_is_owned_by(self, tid)) { - unsigned long count = self->rlock_count + 1; - if (count <= self->rlock_count) { - PyErr_SetString(PyExc_OverflowError, - "Internal lock count overflowed"); - return NULL; - } - self->rlock_count = count; - Py_RETURN_TRUE; - } - r = acquire_timed(self->rlock_lock, timeout); - if (r == PY_LOCK_ACQUIRED) { - assert(self->rlock_count == 0); - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, tid); - self->rlock_count = 1; } - else if (r == PY_LOCK_INTR) { + + PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout, + _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + if (r == PY_LOCK_INTR) { return NULL; } @@ -1086,19 +1034,15 @@ PyDoc_STRVAR(rlock_enter_doc, Lock the lock."); static PyObject * -rlock_release(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_release(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); + rlockobject *self = (rlockobject*)op; - if (!rlock_is_owned_by(self, tid)) { + if (_PyRecursiveMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } - if (--self->rlock_count == 0) { - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); - PyThread_release_lock(self->rlock_lock); - } Py_RETURN_NONE; } @@ -1122,28 +1066,19 @@ PyDoc_STRVAR(rlock_exit_doc, Release the lock."); static PyObject * -rlock_acquire_restore(rlockobject *self, PyObject *args) +rlock_acquire_restore(PyObject *op, PyObject *args) { + rlockobject *self = (rlockobject*)op; PyThread_ident_t owner; - unsigned long count; - int r = 1; + Py_ssize_t count; - if (!PyArg_ParseTuple(args, "(k" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", + if (!PyArg_ParseTuple(args, "(n" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", &count, &owner)) return NULL; - if (!PyThread_acquire_lock(self->rlock_lock, 0)) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock(self->rlock_lock, 1); - Py_END_ALLOW_THREADS - } - if (!r) { - PyErr_SetString(ThreadError, "couldn't acquire lock"); - return NULL; - } - assert(self->rlock_count == 0); - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, owner); - self->rlock_count = count; + _PyRecursiveMutex_Lock(&self->lock); + _Py_atomic_store_ullong_relaxed(&self->lock.thread, owner); + self->lock.level = (size_t)count - 1; Py_RETURN_NONE; } @@ -1154,23 +1089,21 @@ PyDoc_STRVAR(rlock_acquire_restore_doc, For internal use by `threading.Condition`."); static PyObject * -rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_release_save(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t owner; - unsigned long count; + rlockobject *self = (rlockobject*)op; - if (self->rlock_count == 0) { + if (!_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } - owner = self->rlock_owner; - count = self->rlock_count; - self->rlock_count = 0; - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); - PyThread_release_lock(self->rlock_lock); - return Py_BuildValue("k" Py_PARSE_THREAD_IDENT_T, count, owner); + PyThread_ident_t owner = self->lock.thread; + Py_ssize_t count = self->lock.level + 1; + self->lock.level = 0; // ensure the unlock releases the lock + _PyRecursiveMutex_Unlock(&self->lock); + return Py_BuildValue("n" Py_PARSE_THREAD_IDENT_T, count, owner); } PyDoc_STRVAR(rlock_release_save_doc, @@ -1180,12 +1113,13 @@ PyDoc_STRVAR(rlock_release_save_doc, For internal use by `threading.Condition`."); static PyObject * -rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_recursion_count(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - PyThread_ident_t owner = - _Py_atomic_load_ullong_relaxed(&self->rlock_owner); - return PyLong_FromUnsignedLong(owner == tid ? self->rlock_count : 0UL); + rlockobject *self = (rlockobject*)op; + if (_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { + return PyLong_FromSize_t(self->lock.level + 1); + } + return PyLong_FromLong(0); } PyDoc_STRVAR(rlock_recursion_count_doc, @@ -1195,14 +1129,11 @@ PyDoc_STRVAR(rlock_recursion_count_doc, For internal use by reentrancy checks."); static PyObject * -rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_is_owned(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - - if (rlock_is_owned_by(self, tid)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + rlockobject *self = (rlockobject*)op; + long owned = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); + return PyBool_FromLong(owned); } PyDoc_STRVAR(rlock_is_owned_doc, @@ -1218,29 +1149,21 @@ rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (self == NULL) { return NULL; } - self->in_weakreflist = NULL; - self->rlock_owner = 0; - self->rlock_count = 0; - - self->rlock_lock = PyThread_allocate_lock(); - if (self->rlock_lock == NULL) { - Py_DECREF(self); - PyErr_SetString(ThreadError, "can't allocate lock"); - return NULL; - } + self->lock = (_PyRecursiveMutex){0}; return (PyObject *) self; } static PyObject * -rlock_repr(rlockobject *self) +rlock_repr(PyObject *op) { - PyThread_ident_t owner = - _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + rlockobject *self = (rlockobject*)op; + PyThread_ident_t owner = self->lock.thread; + size_t count = self->lock.level + 1; return PyUnicode_FromFormat( - "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%lu at %p>", - self->rlock_count ? "locked" : "unlocked", + "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", + owner ? "locked" : "unlocked", Py_TYPE(self)->tp_name, owner, - self->rlock_count, self); + count, self); } @@ -1248,14 +1171,7 @@ rlock_repr(rlockobject *self) static PyObject * rlock__at_fork_reinit(rlockobject *self, PyObject *Py_UNUSED(args)) { - if (_PyThread_at_fork_reinit(&self->rlock_lock) < 0) { - PyErr_SetString(ThreadError, "failed to reinitialize lock at fork"); - return NULL; - } - - self->rlock_owner = 0; - self->rlock_count = 0; - + self->lock = (_PyRecursiveMutex){0}; Py_RETURN_NONE; } #endif /* HAVE_FORK */ @@ -1264,19 +1180,19 @@ rlock__at_fork_reinit(rlockobject *self, PyObject *Py_UNUSED(args)) static PyMethodDef rlock_methods[] = { {"acquire", _PyCFunction_CAST(rlock_acquire), METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, - {"release", (PyCFunction)rlock_release, + {"release", rlock_release, METH_NOARGS, rlock_release_doc}, - {"_is_owned", (PyCFunction)rlock_is_owned, + {"_is_owned", rlock_is_owned, METH_NOARGS, rlock_is_owned_doc}, - {"_acquire_restore", (PyCFunction)rlock_acquire_restore, + {"_acquire_restore", rlock_acquire_restore, METH_VARARGS, rlock_acquire_restore_doc}, - {"_release_save", (PyCFunction)rlock_release_save, + {"_release_save", rlock_release_save, METH_NOARGS, rlock_release_save_doc}, - {"_recursion_count", (PyCFunction)rlock_recursion_count, + {"_recursion_count", rlock_recursion_count, METH_NOARGS, rlock_recursion_count_doc}, {"__enter__", _PyCFunction_CAST(rlock_acquire), METH_VARARGS | METH_KEYWORDS, rlock_enter_doc}, - {"__exit__", (PyCFunction)rlock_release, + {"__exit__", rlock_release, METH_VARARGS, rlock_exit_doc}, #ifdef HAVE_FORK {"_at_fork_reinit", (PyCFunction)rlock__at_fork_reinit, @@ -1286,18 +1202,12 @@ static PyMethodDef rlock_methods[] = { }; -static PyMemberDef rlock_type_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, in_weakreflist), Py_READONLY}, - {NULL}, -}; - static PyType_Slot rlock_type_slots[] = { - {Py_tp_dealloc, (destructor)rlock_dealloc}, - {Py_tp_repr, (reprfunc)rlock_repr}, + {Py_tp_dealloc, rlock_dealloc}, + {Py_tp_repr, rlock_repr}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_members, rlock_type_members}, {Py_tp_traverse, rlock_traverse}, {0, 0}, }; @@ -1306,7 +1216,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; @@ -1320,16 +1230,7 @@ newlockobject(PyObject *module) if (self == NULL) { return NULL; } - - self->lock_lock = PyThread_allocate_lock(); - self->locked = 0; - self->in_weakreflist = NULL; - - if (self->lock_lock == NULL) { - Py_DECREF(self); - PyErr_SetString(ThreadError, "can't allocate lock"); - return NULL; - } + self->lock = (PyMutex){0}; return self; } @@ -1392,8 +1293,9 @@ typedef struct { } localdummyobject; static void -localdummy_dealloc(localdummyobject *self) +localdummy_dealloc(PyObject *op) { + localdummyobject *self = (localdummyobject*)op; if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); @@ -1407,7 +1309,7 @@ static PyMemberDef local_dummy_type_members[] = { }; static PyType_Slot local_dummy_type_slots[] = { - {Py_tp_dealloc, (destructor)localdummy_dealloc}, + {Py_tp_dealloc, localdummy_dealloc}, {Py_tp_doc, "Thread-local dummy"}, {Py_tp_members, local_dummy_type_members}, {0, 0} diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 4f05cab375ed6b..b0b70ccb8cc3d3 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -143,7 +143,7 @@ _get_tcl_lib_path(void) struct stat stat_buf; int stat_return_value; - PyObject *prefix = PySys_GetObject("prefix"); // borrowed reference + PyObject *prefix = PySys_GetObject("base_prefix"); // borrowed reference if (prefix == NULL) { return NULL; } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 902ece795b575b..c5292575c22f23 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -3,6 +3,7 @@ #endif #include "Python.h" +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -298,15 +299,20 @@ get_weak_cache(zoneinfo_state *state, PyTypeObject *type) } } +/*[clinic input] +@critical_section +@classmethod +zoneinfo.ZoneInfo.__new__ + + key: object + +Create a new ZoneInfo instance. +[clinic start generated code]*/ + static PyObject * -zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) +zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) +/*[clinic end generated code: output=95e61dab86bb95c3 input=ef73d7a83bf8790e]*/ { - PyObject *key = NULL; - static char *kwlist[] = {"key", NULL}; - if (PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &key) == 0) { - return NULL; - } - zoneinfo_state *state = zoneinfo_get_state_by_self(type); PyObject *instance = zone_from_strong_cache(state, type, key); if (instance != NULL || PyErr_Occurred()) { @@ -467,6 +473,7 @@ zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, } /*[clinic input] +@critical_section @classmethod zoneinfo.ZoneInfo.clear_cache @@ -481,7 +488,7 @@ Clear the ZoneInfo cache. static PyObject * zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *only_keys) -/*[clinic end generated code: output=114d9b7c8a22e660 input=e32ca3bb396788ba]*/ +/*[clinic end generated code: output=114d9b7c8a22e660 input=35944715df26d24e]*/ { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); PyObject *weak_cache = get_weak_cache(state, type); @@ -816,14 +823,10 @@ zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, /*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { if (from_cache) { - PyObject *val_args = PyTuple_Pack(1, key); - if (val_args == NULL) { - return NULL; - } - - PyObject *rv = zoneinfo_new(type, val_args, NULL); - - Py_DECREF(val_args); + PyObject *rv; + Py_BEGIN_CRITICAL_SECTION(type); + rv = zoneinfo_ZoneInfo_impl(type, key); + Py_END_CRITICAL_SECTION(); return rv; } else { @@ -858,8 +861,7 @@ load_timedelta(zoneinfo_state *state, long seconds) 0, seconds, 0, 1, PyDateTimeAPI->DeltaType); if (tmp != NULL) { - rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); - Py_XINCREF(rv); + PyDict_SetDefaultRef(state->TIMEDELTA_CACHE, pyoffset, tmp, &rv); Py_DECREF(tmp); } } @@ -2368,6 +2370,7 @@ strong_cache_free(StrongCacheNode *root) static void remove_from_strong_cache(zoneinfo_state *state, StrongCacheNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); if (state->ZONEINFO_STRONG_CACHE == node) { state->ZONEINFO_STRONG_CACHE = node->next; } @@ -2422,6 +2425,7 @@ eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return 0; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { @@ -2478,6 +2482,7 @@ zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return NULL; // Strong cache currently only implemented for base class } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; StrongCacheNode *node = find_in_strong_cache(cache, key); @@ -2504,6 +2509,7 @@ update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *new_node = strong_cache_node_new(key, zone); if (new_node == NULL) { return; @@ -2631,7 +2637,7 @@ static PyType_Slot zoneinfo_slots[] = { {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_methods, zoneinfo_methods}, {Py_tp_members, zoneinfo_members}, - {Py_tp_new, zoneinfo_new}, + {Py_tp_new, zoneinfo_ZoneInfo}, {Py_tp_dealloc, zoneinfo_dealloc}, {Py_tp_traverse, zoneinfo_traverse}, {Py_tp_clear, zoneinfo_clear}, diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 373518673dd352..53135ae4aa7968 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -669,7 +669,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, if (datalen == 0) { ERROR_DECREF(errorcb); - return make_tuple(PyUnicode_New(0, 0), 0); + return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0); } _PyUnicodeWriter_Init(&buf.writer); @@ -1434,7 +1434,7 @@ mbstreamreader_iread(MultibyteStreamReaderObject *self, Py_ssize_t rsize; if (sizehint == 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); _PyUnicodeWriter_Init(&buf.writer); buf.excobj = NULL; diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab350..01855aec5e123e 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -2683,6 +2683,56 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_codecs__unregister_error__doc__, +"_unregister_error($module, errors, /)\n" +"--\n" +"\n" +"Un-register the specified error handler for the error handling `errors\'.\n" +"\n" +"Only custom error handlers can be un-registered. An exception is raised\n" +"if the error handling is a built-in one (e.g., \'strict\'), or if an error\n" +"occurs.\n" +"\n" +"Otherwise, this returns True if a custom handler has been successfully\n" +"un-registered, and False if no custom handler for the specified error\n" +"handling exists."); + +#define _CODECS__UNREGISTER_ERROR_METHODDEF \ + {"_unregister_error", (PyCFunction)_codecs__unregister_error, METH_O, _codecs__unregister_error__doc__}, + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors); + +static PyObject * +_codecs__unregister_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *errors; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_unregister_error", "argument", "str", arg); + goto exit; + } + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + _return_value = _codecs__unregister_error_impl(module, errors); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_codecs_lookup_error__doc__, "lookup_error($module, name, /)\n" "--\n" @@ -2746,4 +2796,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3013d4709d96ffe input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index f7e0aaf7b23649..8399c5620f125b 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -1389,12 +1389,12 @@ _curses_window_overlay(PyCursesWindowObject *self, PyObject *args) switch (PyTuple_GET_SIZE(args)) { case 1: - if (!PyArg_ParseTuple(args, "O!:overlay", &PyCursesWindow_Type, &destwin)) { + if (!PyArg_ParseTuple(args, "O!:overlay", clinic_state()->window_type, &destwin)) { goto exit; } break; case 7: - if (!PyArg_ParseTuple(args, "O!iiiiii:overlay", &PyCursesWindow_Type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { + if (!PyArg_ParseTuple(args, "O!iiiiii:overlay", clinic_state()->window_type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { goto exit; } group_right_1 = 1; @@ -1448,12 +1448,12 @@ _curses_window_overwrite(PyCursesWindowObject *self, PyObject *args) switch (PyTuple_GET_SIZE(args)) { case 1: - if (!PyArg_ParseTuple(args, "O!:overwrite", &PyCursesWindow_Type, &destwin)) { + if (!PyArg_ParseTuple(args, "O!:overwrite", clinic_state()->window_type, &destwin)) { goto exit; } break; case 7: - if (!PyArg_ParseTuple(args, "O!iiiiii:overwrite", &PyCursesWindow_Type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { + if (!PyArg_ParseTuple(args, "O!iiiiii:overwrite", clinic_state()->window_type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { goto exit; } group_right_1 = 1; @@ -3693,25 +3693,55 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, static PyObject * -_curses_resizeterm_impl(PyObject *module, int nlines, int ncols); +_curses_resizeterm_impl(PyObject *module, short nlines, short ncols); static PyObject * _curses_resizeterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int nlines; - int ncols; + short nlines; + short ncols; if (!_PyArg_CheckPositional("resizeterm", nargs, 2, 2)) { goto exit; } - nlines = PyLong_AsInt(args[0]); - if (nlines == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + nlines = (short) ival; + } } - ncols = PyLong_AsInt(args[1]); - if (ncols == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + ncols = (short) ival; + } } return_value = _curses_resizeterm_impl(module, nlines, ncols); @@ -3744,25 +3774,55 @@ PyDoc_STRVAR(_curses_resize_term__doc__, {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, static PyObject * -_curses_resize_term_impl(PyObject *module, int nlines, int ncols); +_curses_resize_term_impl(PyObject *module, short nlines, short ncols); static PyObject * _curses_resize_term(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int nlines; - int ncols; + short nlines; + short ncols; if (!_PyArg_CheckPositional("resize_term", nargs, 2, 2)) { goto exit; } - nlines = PyLong_AsInt(args[0]); - if (nlines == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + nlines = (short) ival; + } } - ncols = PyLong_AsInt(args[1]); - if (ncols == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + ncols = (short) ival; + } } return_value = _curses_resize_term_impl(module, nlines, ncols); @@ -4318,4 +4378,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=96887782374f070a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cd1273354b08948f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 9905b6425e2f79..bde88b5c4fa65b 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -6,8 +6,65 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(zoneinfo_ZoneInfo__doc__, +"ZoneInfo(key)\n" +"--\n" +"\n" +"Create a new ZoneInfo instance."); + +static PyObject * +zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key); + +static PyObject * +zoneinfo_ZoneInfo(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(key), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"key", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "ZoneInfo", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *key; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + key = fastargs[0]; + Py_BEGIN_CRITICAL_SECTION(type); + return_value = zoneinfo_ZoneInfo_impl(type, key); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "from_file($type, file_obj, /, key=None)\n" "--\n" @@ -182,7 +239,9 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *c } only_keys = args[0]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(type); return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, cls, only_keys); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -372,4 +431,4 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con exit: return return_value; } -/*[clinic end generated code: output=2a15f32fdd2ab6cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b4fdc0b30247110a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9722f06a5935b9..d9d919ea75d853 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2015,25 +2015,55 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_exists__doc__, -"_path_exists($module, path, /)\n" +"_path_exists($module, /, path)\n" "--\n" "\n" "Test whether a path exists. Returns False for broken symbolic links."); #define OS__PATH_EXISTS_METHODDEF \ - {"_path_exists", (PyCFunction)os__path_exists, METH_O, os__path_exists__doc__}, + {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, static int os__path_exists_impl(PyObject *module, path_t *path); static PyObject * -os__path_exists(PyObject *module, PyObject *arg) +os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_exists", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); int _return_value; - if (!path_converter(arg, &path)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { goto exit; } _return_value = os__path_exists_impl(module, &path); @@ -2054,25 +2084,55 @@ os__path_exists(PyObject *module, PyObject *arg) #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_lexists__doc__, -"_path_lexists($module, path, /)\n" +"_path_lexists($module, /, path)\n" "--\n" "\n" "Test whether a path exists. Returns True for broken symbolic links."); #define OS__PATH_LEXISTS_METHODDEF \ - {"_path_lexists", (PyCFunction)os__path_lexists, METH_O, os__path_lexists__doc__}, + {"_path_lexists", _PyCFunction_CAST(os__path_lexists), METH_FASTCALL|METH_KEYWORDS, os__path_lexists__doc__}, static int os__path_lexists_impl(PyObject *module, path_t *path); static PyObject * -os__path_lexists(PyObject *module, PyObject *arg) +os__path_lexists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_lexists", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); int _return_value; - if (!path_converter(arg, &path)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { goto exit; } _return_value = os__path_lexists_impl(module, &path); @@ -5954,7 +6014,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(__linux__) && defined(__NR_pidfd_open)) +#if (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -6013,7 +6073,7 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_open)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #if defined(HAVE_SETNS) @@ -12837,4 +12897,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=a736ad3f7205176e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=18d75b737513dae6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 1d3a143dfd8d39..986c0289f2bfcb 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -670,7 +670,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PTHREAD_KILL) */ -#if (defined(__linux__) && defined(__NR_pidfd_send_signal)) +#if (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(signal_pidfd_send_signal__doc__, "pidfd_send_signal($module, pidfd, signalnum, siginfo=None, flags=0, /)\n" @@ -723,7 +723,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #ifndef SIGNAL_ALARM_METHODDEF #define SIGNAL_ALARM_METHODDEF @@ -776,4 +776,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=6d8e17a32cef668f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c57b4b98fad6f4b8 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index e740ec4d7625c3..1201fa094902d7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1036,7 +1036,7 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) /*[clinic end generated code: output=1c64519cd859c2f0 input=c99a1472c425d66d]*/ { Py_ssize_t i; - PyObject *it, *copyable, *copyfunc, *result; + PyObject *it, *to, *result; if (n < 0) { PyErr_SetString(PyExc_ValueError, "n must be >= 0"); @@ -1053,41 +1053,23 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) return NULL; } - if (PyObject_GetOptionalAttr(it, &_Py_ID(__copy__), ©func) < 0) { - Py_DECREF(it); + itertools_state *state = get_module_state(module); + to = tee_fromiterable(state, it); + Py_DECREF(it); + if (to == NULL) { Py_DECREF(result); return NULL; } - if (copyfunc != NULL) { - copyable = it; - } - else { - itertools_state *state = get_module_state(module); - copyable = tee_fromiterable(state, it); - Py_DECREF(it); - if (copyable == NULL) { - Py_DECREF(result); - return NULL; - } - copyfunc = PyObject_GetAttr(copyable, &_Py_ID(__copy__)); - if (copyfunc == NULL) { - Py_DECREF(copyable); - Py_DECREF(result); - return NULL; - } - } - PyTuple_SET_ITEM(result, 0, copyable); + PyTuple_SET_ITEM(result, 0, to); for (i = 1; i < n; i++) { - copyable = _PyObject_CallNoArgs(copyfunc); - if (copyable == NULL) { - Py_DECREF(copyfunc); + to = tee_copy((teeobject *)to, NULL); + if (to == NULL) { Py_DECREF(result); return NULL; } - PyTuple_SET_ITEM(result, i, copyable); + PyTuple_SET_ITEM(result, i, to); } - Py_DECREF(copyfunc); return result; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index baf2dc439b8959..058f57770755aa 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1657,7 +1657,7 @@ math_isqrt(PyObject *module, PyObject *n) /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ { int a_too_large, c_bit_length; - uint64_t c, d; + int64_t c, d; uint64_t m; uint32_t u; PyObject *a = NULL, *b; @@ -1680,14 +1680,13 @@ math_isqrt(PyObject *module, PyObject *n) /* c = (n.bit_length() - 1) // 2 */ c = _PyLong_NumBits(n); - if (c == (uint64_t)(-1)) { - goto error; - } - c = (c - 1U) / 2U; + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a fast, almost branch-free algorithm. */ - if (c <= 31U) { + if (c <= 31) { int shift = 31 - (int)c; m = (uint64_t)PyLong_AsUnsignedLongLong(n); Py_DECREF(n); @@ -1704,13 +1703,13 @@ math_isqrt(PyObject *module, PyObject *n) /* From n >= 2**64 it follows that c.bit_length() >= 6. */ c_bit_length = 6; - while ((c >> c_bit_length) > 0U) { + while ((c >> c_bit_length) > 0) { ++c_bit_length; } /* Initialise d and a. */ d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2U*c - 62U); + b = _PyLong_Rshift(n, 2*c - 62); if (b == NULL) { goto error; } @@ -1727,12 +1726,12 @@ math_isqrt(PyObject *module, PyObject *n) for (int s = c_bit_length - 6; s >= 0; --s) { PyObject *q; - uint64_t e = d; + int64_t e = d; d = c >> s; /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2U*c - d - e + 1U); + q = _PyLong_Rshift(n, 2*c - d - e + 1); if (q == NULL) { goto error; } @@ -1742,7 +1741,7 @@ math_isqrt(PyObject *module, PyObject *n) } /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e)); + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); if (a == NULL) { Py_DECREF(q); goto error; @@ -2202,8 +2201,8 @@ loghelper(PyObject* arg, double (*func)(double)) to compute the log anyway. Clear the exception and continue. */ PyErr_Clear(); x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; + assert(e >= 0); + assert(!PyErr_Occurred()); /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ result = func(x) + func(2.0) * e; } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 99a85e9e49ad47..e1c26e19932664 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -486,7 +486,7 @@ mmap_read_line_method(mmap_object *self, remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (!remaining) - return PyBytes_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); start = self->data + self->pos; if (safe_memchr(&eol, start, '\n', remaining) < 0) { @@ -1274,7 +1274,7 @@ mmap_subscript(mmap_object *self, PyObject *item) CHECK_VALID(NULL); if (slicelen <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); else if (step == 1) return _safe_PyBytes_FromStringAndSize(self->data + start, slicelen); else { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 86366c66c46552..c0af78ba075e85 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5391,7 +5391,6 @@ _testFileType(path_t *path, int testedType) os._path_exists -> bool path: path_t(allow_fd=True, suppress_value_error=True) - / Test whether a path exists. Returns False for broken symbolic links. @@ -5399,7 +5398,7 @@ Test whether a path exists. Returns False for broken symbolic links. static int os__path_exists_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=8da13acf666e16ba input=29198507a6082a57]*/ +/*[clinic end generated code: output=8da13acf666e16ba input=142beabfc66783eb]*/ { return _testFileExists(path, TRUE); } @@ -5409,7 +5408,6 @@ os__path_exists_impl(PyObject *module, path_t *path) os._path_lexists -> bool path: path_t(allow_fd=True, suppress_value_error=True) - / Test whether a path exists. Returns True for broken symbolic links. @@ -5417,7 +5415,7 @@ Test whether a path exists. Returns True for broken symbolic links. static int os__path_lexists_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=e7240ed5fc45bff3 input=03d9fed8bc6ce96f]*/ +/*[clinic end generated code: output=e7240ed5fc45bff3 input=208205112a3cc1ed]*/ { return _testFileExists(path, FALSE); } @@ -10121,7 +10119,10 @@ os_wait_impl(PyObject *module) } #endif /* HAVE_WAIT */ -#if defined(__linux__) && defined(__NR_pidfd_open) + +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_open) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] os.pidfd_open pid: pid_t diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 73bfcb756657b8..0e53a36bca55f0 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1299,7 +1299,9 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, #endif /* #if defined(HAVE_PTHREAD_KILL) */ -#if defined(__linux__) && defined(__NR_pidfd_send_signal) +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_send_signal) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] signal.pidfd_send_signal diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index ded6f255aaddea..744e5e0c0b2b54 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5636,7 +5636,7 @@ socket_gethostname(PyObject *self, PyObject *unused) return PyErr_SetFromWindowsErr(0); if (size == 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* MSDN says ERROR_MORE_DATA may occur because DNS allows longer names */ @@ -7921,6 +7921,9 @@ socket_exec(PyObject *m) #ifdef SO_OOBINLINE ADD_INT_MACRO(m, SO_OOBINLINE); #endif +#ifdef SO_ORIGINAL_DST + ADD_INT_MACRO(m, SO_ORIGINAL_DST); +#endif #ifndef __GNU__ #ifdef SO_REUSEPORT ADD_INT_MACRO(m, SO_REUSEPORT); diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index a77c620c2ef630..e1d96377728eb9 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -172,6 +172,10 @@ typedef int socklen_t; # undef AF_VSOCK #endif +#ifdef HAVE_LINUX_NETFILTER_IPV4_H +# include +#endif + #ifdef HAVE_SOCKADDR_ALG # include diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ee59fb73ac1e31..9720c201a184a8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -813,7 +813,12 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +// Some platforms only support a limited range of years. +// +// Android works with negative years on the emulator, but fails on some +// physical devices (#123017). +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ + || defined(__VXWORKS__) || defined(__ANDROID__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 333ffe68a454e4..60bde755d24574 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -413,7 +413,7 @@ unicodedata_UCD_decomposition_impl(PyObject *self, int chr) if (UCD_Check(self)) { const change_record *old = get_old_record(self, c); if (old->category_changed == 0) - return PyUnicode_FromString(""); /* unassigned */ + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* unassigned */ } if (code < 0 || code >= 0x110000) diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index c5aaf22eeb2948..78dcce73cdaade 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -267,12 +267,12 @@ newcompobject(PyTypeObject *type) self->eof = 0; self->is_initialised = 0; self->zdict = NULL; - self->unused_data = PyBytes_FromStringAndSize("", 0); + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->unused_data == NULL) { Py_DECREF(self); return NULL; } - self->unconsumed_tail = PyBytes_FromStringAndSize("", 0); + self->unconsumed_tail = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->unconsumed_tail == NULL) { Py_DECREF(self); return NULL; diff --git a/Objects/abstract.c b/Objects/abstract.c index 7cca81464cd112..f6647874d732f6 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -862,7 +862,7 @@ PyObject_Format(PyObject *obj, PyObject *format_spec) /* If no format_spec is provided, use an empty string */ if (format_spec == NULL) { - empty = PyUnicode_New(0, 0); + empty = Py_GetConstant(Py_CONSTANT_EMPTY_STR); format_spec = empty; } diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a80e4670665a22..fd2a85a3fe0a61 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -42,15 +42,16 @@ _getbytevalue(PyObject* arg, int *value) } static int -bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) +bytearray_getbuffer(PyObject *self, Py_buffer *view, int flags) { - void *ptr; + PyByteArrayObject *obj = _PyByteArray_CAST(self); if (view == NULL) { PyErr_SetString(PyExc_BufferError, "bytearray_getbuffer: view==NULL argument is obsolete"); return -1; } - ptr = (void *) PyByteArray_AS_STRING(obj); + + void *ptr = (void *) PyByteArray_AS_STRING(obj); /* cannot fail if view != NULL and readonly == 0 */ (void)PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags); obj->ob_exports++; @@ -58,8 +59,9 @@ bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) } static void -bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) +bytearray_releasebuffer(PyObject *self, Py_buffer *view) { + PyByteArrayObject *obj = _PyByteArray_CAST(self); obj->ob_exports--; assert(obj->ob_exports >= 0); } @@ -286,46 +288,53 @@ PyByteArray_Concat(PyObject *a, PyObject *b) /* Functions stuffed into the type object */ static Py_ssize_t -bytearray_length(PyByteArrayObject *self) +bytearray_length(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); return Py_SIZE(self); } static PyObject * -bytearray_iconcat(PyByteArrayObject *self, PyObject *other) +bytearray_iconcat(PyObject *op, PyObject *other) { - Py_ssize_t size; - Py_buffer vo; + PyByteArrayObject *self = _PyByteArray_CAST(op); + Py_buffer vo; if (PyObject_GetBuffer(other, &vo, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name); return NULL; } - size = Py_SIZE(self); + Py_ssize_t size = Py_SIZE(self); if (size > PY_SSIZE_T_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } + if (PyByteArray_Resize((PyObject *)self, size + vo.len) < 0) { PyBuffer_Release(&vo); return NULL; } + memcpy(PyByteArray_AS_STRING(self) + size, vo.buf, vo.len); PyBuffer_Release(&vo); return Py_NewRef(self); } static PyObject * -bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count) +bytearray_repeat(PyObject *op, Py_ssize_t count) { - if (count < 0) + PyByteArrayObject *self = _PyByteArray_CAST(op); + if (count < 0) { count = 0; + } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) + if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { return PyErr_NoMemory(); + } Py_ssize_t size = mysize * count; + PyByteArrayObject* result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size); const char* buf = PyByteArray_AS_STRING(self); if (result != NULL && size != 0) { @@ -335,20 +344,24 @@ bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count) } static PyObject * -bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) +bytearray_irepeat(PyObject *op, Py_ssize_t count) { - if (count < 0) + PyByteArrayObject *self = _PyByteArray_CAST(op); + if (count < 0) { count = 0; + } else if (count == 1) { return Py_NewRef(self); } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) + if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { return PyErr_NoMemory(); + } const Py_ssize_t size = mysize * count; - if (PyByteArray_Resize((PyObject *)self, size) < 0) + if (PyByteArray_Resize((PyObject *)self, size) < 0) { return NULL; + } char* buf = PyByteArray_AS_STRING(self); _PyBytes_Repeat(buf, size, buf, mysize); @@ -357,8 +370,9 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) } static PyObject * -bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) +bytearray_getitem(PyObject *op, Py_ssize_t i) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (i < 0 || i >= Py_SIZE(self)) { PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); return NULL; @@ -367,8 +381,9 @@ bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) } static PyObject * -bytearray_subscript(PyByteArrayObject *self, PyObject *index) +bytearray_subscript(PyObject *op, PyObject *index) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -559,12 +574,13 @@ bytearray_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, } static int -bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) +bytearray_setitem(PyObject *op, Py_ssize_t i, PyObject *value) { - int ival = -1; + PyByteArrayObject *self = _PyByteArray_CAST(op); // GH-91153: We need to do this *before* the size check, in case value has a // nasty __index__ method that changes the size of the bytearray: + int ival = -1; if (value && !_getbytevalue(value, &ival)) { return -1; } @@ -588,11 +604,11 @@ bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) } static int -bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *values) +bytearray_ass_subscript(PyObject *op, PyObject *index, PyObject *values) { - Py_ssize_t start, stop, step, slicelen, needed; - char *buf, *bytes; - buf = PyByteArray_AS_STRING(self); + PyByteArrayObject *self = _PyByteArray_CAST(op); + Py_ssize_t start, stop, step, slicelen; + char *buf = PyByteArray_AS_STRING(self); if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -645,6 +661,8 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu return -1; } + char *bytes; + Py_ssize_t needed; if (values == NULL) { bytes = NULL; needed = 0; @@ -661,7 +679,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu values = PyByteArray_FromObject(values); if (values == NULL) return -1; - err = bytearray_ass_subscript(self, index, values); + err = bytearray_ass_subscript((PyObject*)self, index, values); Py_DECREF(values); return err; } @@ -670,10 +688,14 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu bytes = PyByteArray_AS_STRING(values); needed = Py_SIZE(values); } + /* Make sure b[5:2] = ... inserts before 5, not before 2. */ if ((step < 0 && start < stop) || (step > 0 && start > stop)) + { stop = start; + } + if (step == 1) { return bytearray_setslice_linear(self, start, stop, bytes, needed); } @@ -785,7 +807,7 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, if (encoded == NULL) return -1; assert(PyBytes_Check(encoded)); - new = bytearray_iconcat(self, encoded); + new = bytearray_iconcat((PyObject*)self, encoded); Py_DECREF(encoded); if (new == NULL) return -1; @@ -926,8 +948,9 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, /* Mostly copied from string_repr, but without the "smart quote" functionality. */ static PyObject * -bytearray_repr(PyByteArrayObject *self) +bytearray_repr(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); const char *className = _PyType_Name(Py_TYPE(self)); const char *quote_prefix = "(b"; const char *quote_postfix = ")"; @@ -1021,7 +1044,7 @@ bytearray_str(PyObject *op) return NULL; } } - return bytearray_repr((PyByteArrayObject*)op); + return bytearray_repr(op); } static PyObject * @@ -1080,8 +1103,9 @@ bytearray_richcompare(PyObject *self, PyObject *other, int op) } static void -bytearray_dealloc(PyByteArrayObject *self) +bytearray_dealloc(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (self->ob_exports > 0) { PyErr_SetString(PyExc_SystemError, "deallocated bytearray object has exported buffers"); @@ -1244,7 +1268,9 @@ bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, static int bytearray_contains(PyObject *self, PyObject *arg) { - return _Py_bytes_contains(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), arg); + return _Py_bytes_contains(PyByteArray_AS_STRING(self), + PyByteArray_GET_SIZE(self), + arg); } /*[clinic input] @@ -2262,31 +2288,30 @@ bytearray_sizeof_impl(PyByteArrayObject *self) } static PySequenceMethods bytearray_as_sequence = { - (lenfunc)bytearray_length, /* sq_length */ - (binaryfunc)PyByteArray_Concat, /* sq_concat */ - (ssizeargfunc)bytearray_repeat, /* sq_repeat */ - (ssizeargfunc)bytearray_getitem, /* sq_item */ + bytearray_length, /* sq_length */ + PyByteArray_Concat, /* sq_concat */ + bytearray_repeat, /* sq_repeat */ + bytearray_getitem, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)bytearray_setitem, /* sq_ass_item */ + bytearray_setitem, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)bytearray_contains, /* sq_contains */ - (binaryfunc)bytearray_iconcat, /* sq_inplace_concat */ - (ssizeargfunc)bytearray_irepeat, /* sq_inplace_repeat */ + bytearray_contains, /* sq_contains */ + bytearray_iconcat, /* sq_inplace_concat */ + bytearray_irepeat, /* sq_inplace_repeat */ }; static PyMappingMethods bytearray_as_mapping = { - (lenfunc)bytearray_length, - (binaryfunc)bytearray_subscript, - (objobjargproc)bytearray_ass_subscript, + bytearray_length, + bytearray_subscript, + bytearray_ass_subscript, }; static PyBufferProcs bytearray_as_buffer = { - (getbufferproc)bytearray_getbuffer, - (releasebufferproc)bytearray_releasebuffer, + bytearray_getbuffer, + bytearray_releasebuffer, }; -static PyMethodDef -bytearray_methods[] = { +static PyMethodDef bytearray_methods[] = { {"__alloc__", (PyCFunction)bytearray_alloc, METH_NOARGS, alloc_doc}, BYTEARRAY_REDUCE_METHODDEF BYTEARRAY_REDUCE_EX_METHODDEF @@ -2391,12 +2416,12 @@ PyTypeObject PyByteArray_Type = { "bytearray", sizeof(PyByteArrayObject), 0, - (destructor)bytearray_dealloc, /* tp_dealloc */ + bytearray_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)bytearray_repr, /* tp_repr */ + bytearray_repr, /* tp_repr */ &bytearray_as_number, /* tp_as_number */ &bytearray_as_sequence, /* tp_as_sequence */ &bytearray_as_mapping, /* tp_as_mapping */ @@ -2411,7 +2436,7 @@ PyTypeObject PyByteArray_Type = { bytearray_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)bytearray_richcompare, /* tp_richcompare */ + bytearray_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ bytearray_iter, /* tp_iter */ 0, /* tp_iternext */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ba6636808d90e0..dcc1aba76abbed 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -46,7 +46,7 @@ Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, static inline PyObject* bytes_get_empty(void) { PyObject *empty = &EMPTY->ob_base.ob_base; - assert(_Py_IsImmortalLoose(empty)); + assert(_Py_IsImmortal(empty)); return empty; } @@ -119,7 +119,7 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) } if (size == 1 && str != NULL) { op = CHARACTER(*str & 255); - assert(_Py_IsImmortalLoose(op)); + assert(_Py_IsImmortal(op)); return (PyObject *)op; } if (size == 0) { @@ -155,7 +155,7 @@ PyBytes_FromString(const char *str) } else if (size == 1) { op = CHARACTER(*str & 255); - assert(_Py_IsImmortalLoose(op)); + assert(_Py_IsImmortal(op)); return (PyObject *)op; } @@ -1405,8 +1405,9 @@ bytes_str(PyObject *op) } static Py_ssize_t -bytes_length(PyBytesObject *a) +bytes_length(PyObject *self) { + PyBytesObject *a = _PyBytes_CAST(self); return Py_SIZE(a); } @@ -1456,11 +1457,9 @@ bytes_concat(PyObject *a, PyObject *b) } static PyObject * -bytes_repeat(PyBytesObject *a, Py_ssize_t n) +bytes_repeat(PyObject *self, Py_ssize_t n) { - Py_ssize_t size; - PyBytesObject *op; - size_t nbytes; + PyBytesObject *a = _PyBytes_CAST(self); if (n < 0) n = 0; /* watch out for overflows: the size can overflow int, @@ -1471,17 +1470,17 @@ bytes_repeat(PyBytesObject *a, Py_ssize_t n) "repeated bytes are too long"); return NULL; } - size = Py_SIZE(a) * n; + Py_ssize_t size = Py_SIZE(a) * n; if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { return Py_NewRef(a); } - nbytes = (size_t)size; + size_t nbytes = (size_t)size; if (nbytes + PyBytesObject_SIZE <= nbytes) { PyErr_SetString(PyExc_OverflowError, "repeated bytes are too long"); return NULL; } - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + nbytes); + PyBytesObject *op = PyObject_Malloc(PyBytesObject_SIZE + nbytes); if (op == NULL) { return PyErr_NoMemory(); } @@ -1504,8 +1503,9 @@ bytes_contains(PyObject *self, PyObject *arg) } static PyObject * -bytes_item(PyBytesObject *a, Py_ssize_t i) +bytes_item(PyObject *self, Py_ssize_t i) { + PyBytesObject *a = _PyBytes_CAST(self); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -1531,21 +1531,17 @@ bytes_compare_eq(PyBytesObject *a, PyBytesObject *b) } static PyObject* -bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) +bytes_richcompare(PyObject *aa, PyObject *bb, int op) { - int c; - Py_ssize_t len_a, len_b; - Py_ssize_t min_len; - /* Make sure both arguments are strings. */ - if (!(PyBytes_Check(a) && PyBytes_Check(b))) { + if (!(PyBytes_Check(aa) && PyBytes_Check(bb))) { if (_Py_GetConfig()->bytes_warning && (op == Py_EQ || op == Py_NE)) { - if (PyUnicode_Check(a) || PyUnicode_Check(b)) { + if (PyUnicode_Check(aa) || PyUnicode_Check(bb)) { if (PyErr_WarnEx(PyExc_BytesWarning, "Comparison between bytes and string", 1)) return NULL; } - if (PyLong_Check(a) || PyLong_Check(b)) { + if (PyLong_Check(aa) || PyLong_Check(bb)) { if (PyErr_WarnEx(PyExc_BytesWarning, "Comparison between bytes and int", 1)) return NULL; @@ -1553,7 +1549,10 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) } Py_RETURN_NOTIMPLEMENTED; } - else if (a == b) { + + PyBytesObject *a = _PyBytes_CAST(aa); + PyBytesObject *b = _PyBytes_CAST(bb); + if (a == b) { switch (op) { case Py_EQ: case Py_LE: @@ -1575,25 +1574,29 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) return PyBool_FromLong(eq); } else { - len_a = Py_SIZE(a); - len_b = Py_SIZE(b); - min_len = Py_MIN(len_a, len_b); + Py_ssize_t len_a = Py_SIZE(a); + Py_ssize_t len_b = Py_SIZE(b); + Py_ssize_t min_len = Py_MIN(len_a, len_b); + int c; if (min_len > 0) { c = Py_CHARMASK(*a->ob_sval) - Py_CHARMASK(*b->ob_sval); if (c == 0) c = memcmp(a->ob_sval, b->ob_sval, min_len); } - else + else { c = 0; - if (c != 0) + } + if (c != 0) { Py_RETURN_RICHCOMPARE(c, 0, op); + } Py_RETURN_RICHCOMPARE(len_a, len_b, op); } } static Py_hash_t -bytes_hash(PyBytesObject *a) +bytes_hash(PyObject *self) { + PyBytesObject *a = _PyBytes_CAST(self); _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS if (a->ob_shash == -1) { @@ -1605,8 +1608,9 @@ _Py_COMP_DIAG_POP } static PyObject* -bytes_subscript(PyBytesObject* self, PyObject* item) +bytes_subscript(PyObject *op, PyObject* item) { + PyBytesObject *self = _PyBytes_CAST(op); if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -1634,7 +1638,7 @@ bytes_subscript(PyBytesObject* self, PyObject* item) &stop, step); if (slicelength <= 0) { - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else if (start == 0 && step == 1 && slicelength == PyBytes_GET_SIZE(self) && @@ -1670,31 +1674,32 @@ bytes_subscript(PyBytesObject* self, PyObject* item) } static int -bytes_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags) +bytes_buffer_getbuffer(PyObject *op, Py_buffer *view, int flags) { + PyBytesObject *self = _PyBytes_CAST(op); return PyBuffer_FillInfo(view, (PyObject*)self, (void *)self->ob_sval, Py_SIZE(self), 1, flags); } static PySequenceMethods bytes_as_sequence = { - (lenfunc)bytes_length, /*sq_length*/ - (binaryfunc)bytes_concat, /*sq_concat*/ - (ssizeargfunc)bytes_repeat, /*sq_repeat*/ - (ssizeargfunc)bytes_item, /*sq_item*/ + bytes_length, /*sq_length*/ + bytes_concat, /*sq_concat*/ + bytes_repeat, /*sq_repeat*/ + bytes_item, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ 0, /*sq_ass_slice*/ - (objobjproc)bytes_contains /*sq_contains*/ + bytes_contains /*sq_contains*/ }; static PyMappingMethods bytes_as_mapping = { - (lenfunc)bytes_length, - (binaryfunc)bytes_subscript, + bytes_length, + bytes_subscript, 0, }; static PyBufferProcs bytes_as_buffer = { - (getbufferproc)bytes_buffer_getbuffer, + bytes_buffer_getbuffer, NULL, }; @@ -3043,11 +3048,11 @@ PyTypeObject PyBytes_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)bytes_repr, /* tp_repr */ + bytes_repr, /* tp_repr */ &bytes_as_number, /* tp_as_number */ &bytes_as_sequence, /* tp_as_sequence */ &bytes_as_mapping, /* tp_as_mapping */ - (hashfunc)bytes_hash, /* tp_hash */ + bytes_hash, /* tp_hash */ 0, /* tp_call */ bytes_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ diff --git a/Objects/capsule.c b/Objects/capsule.c index 555979dab2b789..28965e0f21b7a0 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -317,10 +317,14 @@ static int capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) { // Capsule object is only tracked by the GC - // if _PyCapsule_SetTraverse() is called - assert(capsule->traverse_func != NULL); + // if _PyCapsule_SetTraverse() is called, but + // this can still be manually triggered by gc.get_referents() + + if (capsule->traverse_func != NULL) { + return capsule->traverse_func((PyObject*)capsule, visit, arg); + } - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return 0; } diff --git a/Objects/cellobject.c b/Objects/cellobject.c index b1154e4ca4ace6..590c8a80857699 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -5,6 +5,8 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#define _PyCell_CAST(op) _Py_CAST(PyCellObject*, (op)) + PyObject * PyCell_New(PyObject *obj) { @@ -72,8 +74,9 @@ PyCell_Set(PyObject *op, PyObject *value) } static void -cell_dealloc(PyCellObject *op) +cell_dealloc(PyObject *self) { + PyCellObject *op = _PyCell_CAST(self); _PyObject_GC_UNTRACK(op); Py_XDECREF(op->ob_ref); PyObject_GC_Del(op); @@ -100,10 +103,12 @@ cell_richcompare(PyObject *a, PyObject *b, int op) } static PyObject * -cell_repr(PyCellObject *op) +cell_repr(PyObject *self) { - if (op->ob_ref == NULL) + PyCellObject *op = _PyCell_CAST(self); + if (op->ob_ref == NULL) { return PyUnicode_FromFormat("", op); + } return PyUnicode_FromFormat("", op, Py_TYPE(op->ob_ref)->tp_name, @@ -111,24 +116,26 @@ cell_repr(PyCellObject *op) } static int -cell_traverse(PyCellObject *op, visitproc visit, void *arg) +cell_traverse(PyObject *self, visitproc visit, void *arg) { + PyCellObject *op = _PyCell_CAST(self); Py_VISIT(op->ob_ref); return 0; } static int -cell_clear(PyCellObject *op) +cell_clear(PyObject *self) { + PyCellObject *op = _PyCell_CAST(self); Py_CLEAR(op->ob_ref); return 0; } static PyObject * -cell_get_contents(PyCellObject *op, void *closure) +cell_get_contents(PyObject *self, void *closure) { - if (op->ob_ref == NULL) - { + PyCellObject *op = _PyCell_CAST(self); + if (op->ob_ref == NULL) { PyErr_SetString(PyExc_ValueError, "Cell is empty"); return NULL; } @@ -136,15 +143,15 @@ cell_get_contents(PyCellObject *op, void *closure) } static int -cell_set_contents(PyCellObject *op, PyObject *obj, void *Py_UNUSED(ignored)) +cell_set_contents(PyObject *self, PyObject *obj, void *Py_UNUSED(ignored)) { + PyCellObject *op = _PyCell_CAST(self); Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); return 0; } static PyGetSetDef cell_getsetlist[] = { - {"cell_contents", (getter)cell_get_contents, - (setter)cell_set_contents, NULL}, + {"cell_contents", cell_get_contents, cell_set_contents, NULL}, {NULL} /* sentinel */ }; @@ -153,12 +160,12 @@ PyTypeObject PyCell_Type = { "cell", sizeof(PyCellObject), 0, - (destructor)cell_dealloc, /* tp_dealloc */ + cell_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)cell_repr, /* tp_repr */ + cell_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -170,8 +177,8 @@ PyTypeObject PyCell_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ cell_new_doc, /* tp_doc */ - (traverseproc)cell_traverse, /* tp_traverse */ - (inquiry)cell_clear, /* tp_clear */ + cell_traverse, /* tp_traverse */ + cell_clear, /* tp_clear */ cell_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -186,6 +193,6 @@ PyTypeObject PyCell_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - (newfunc)cell_new, /* tp_new */ + cell_new, /* tp_new */ 0, /* tp_free */ }; diff --git a/Objects/classobject.c b/Objects/classobject.c index 69a7d5f046e30d..775894ad5a7166 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -10,6 +10,7 @@ #include "clinic/classobject.c.h" +#define _PyMethodObject_CAST(op) _Py_CAST(PyMethodObject*, (op)) #define TP_DESCR_GET(t) ((t)->tp_descr_get) /*[clinic input] @@ -166,13 +167,14 @@ static PyMemberDef method_memberlist[] = { should only be used for the class, not for instances */ static PyObject * -method_get_doc(PyMethodObject *im, void *context) +method_get_doc(PyObject *self, void *context) { + PyMethodObject *im = _PyMethodObject_CAST(self); return PyObject_GetAttr(im->im_func, &_Py_ID(__doc__)); } static PyGetSetDef method_getset[] = { - {"__doc__", (getter)method_get_doc, NULL, NULL}, + {"__doc__", method_get_doc, NULL, NULL}, {0} }; @@ -235,8 +237,9 @@ method_new_impl(PyTypeObject *type, PyObject *function, PyObject *instance) } static void -method_dealloc(PyMethodObject *im) +method_dealloc(PyObject *self) { + PyMethodObject *im = _PyMethodObject_CAST(self); _PyObject_GC_UNTRACK(im); if (im->im_weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)im); @@ -274,8 +277,9 @@ method_richcompare(PyObject *self, PyObject *other, int op) } static PyObject * -method_repr(PyMethodObject *a) +method_repr(PyObject *op) { + PyMethodObject *a = _PyMethodObject_CAST(op); PyObject *self = a->im_self; PyObject *func = a->im_func; PyObject *funcname, *result; @@ -301,22 +305,26 @@ method_repr(PyMethodObject *a) } static Py_hash_t -method_hash(PyMethodObject *a) +method_hash(PyObject *self) { - Py_hash_t x, y; - x = PyObject_GenericHash(a->im_self); - y = PyObject_Hash(a->im_func); - if (y == -1) + PyMethodObject *a = _PyMethodObject_CAST(self); + Py_hash_t x = PyObject_GenericHash(a->im_self); + Py_hash_t y = PyObject_Hash(a->im_func); + if (y == -1) { return -1; + } + x = x ^ y; - if (x == -1) + if (x == -1) { x = -2; + } return x; } static int -method_traverse(PyMethodObject *im, visitproc visit, void *arg) +method_traverse(PyObject *self, visitproc visit, void *arg) { + PyMethodObject *im = _PyMethodObject_CAST(self); Py_VISIT(im->im_func); Py_VISIT(im->im_self); return 0; @@ -333,17 +341,17 @@ PyTypeObject PyMethod_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "method", .tp_basicsize = sizeof(PyMethodObject), - .tp_dealloc = (destructor)method_dealloc, + .tp_dealloc = method_dealloc, .tp_vectorcall_offset = offsetof(PyMethodObject, vectorcall), - .tp_repr = (reprfunc)method_repr, - .tp_hash = (hashfunc)method_hash, + .tp_repr = method_repr, + .tp_hash = method_hash, .tp_call = PyVectorcall_Call, .tp_getattro = method_getattro, .tp_setattro = PyObject_GenericSetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL, .tp_doc = method_new__doc__, - .tp_traverse = (traverseproc)method_traverse, + .tp_traverse = method_traverse, .tp_richcompare = method_richcompare, .tp_weaklistoffset = offsetof(PyMethodObject, im_weakreflist), .tp_methods = method_methods, @@ -399,7 +407,7 @@ instancemethod_get_doc(PyObject *self, void *context) } static PyGetSetDef instancemethod_getset[] = { - {"__doc__", (getter)instancemethod_get_doc, NULL, NULL}, + {"__doc__", instancemethod_get_doc, NULL, NULL}, {0} }; @@ -537,7 +545,7 @@ PyTypeObject PyInstanceMethod_Type = { .tp_name = "instancemethod", .tp_basicsize = sizeof(PyInstanceMethodObject), .tp_dealloc = instancemethod_dealloc, - .tp_repr = (reprfunc)instancemethod_repr, + .tp_repr = instancemethod_repr, .tp_call = instancemethod_call, .tp_getattro = instancemethod_getattro, .tp_setattro = PyObject_GenericSetAttr, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 6f0b3f8b9a3262..de80f6cca2904f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1278,8 +1278,9 @@ typedef struct { static void -lineiter_dealloc(lineiterator *li) +lineiter_dealloc(PyObject *self) { + lineiterator *li = (lineiterator*)self; Py_DECREF(li->li_code); Py_TYPE(li)->tp_free(li); } @@ -1293,8 +1294,9 @@ _source_offset_converter(int *value) { } static PyObject * -lineiter_next(lineiterator *li) +lineiter_next(PyObject *self) { + lineiterator *li = (lineiterator*)self; PyCodeAddressRange *bounds = &li->li_line; if (!_PyLineTable_NextAddressRange(bounds)) { return NULL; @@ -1318,7 +1320,7 @@ PyTypeObject _PyLineIterator = { sizeof(lineiterator), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)lineiter_dealloc, /* tp_dealloc */ + lineiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1340,7 +1342,7 @@ PyTypeObject _PyLineIterator = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)lineiter_next, /* tp_iternext */ + lineiter_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1379,15 +1381,17 @@ typedef struct { } positionsiterator; static void -positionsiter_dealloc(positionsiterator* pi) +positionsiter_dealloc(PyObject *self) { + positionsiterator *pi = (positionsiterator*)self; Py_DECREF(pi->pi_code); Py_TYPE(pi)->tp_free(pi); } static PyObject* -positionsiter_next(positionsiterator* pi) +positionsiter_next(PyObject *self) { + positionsiterator *pi = (positionsiterator*)self; if (pi->pi_offset >= pi->pi_range.ar_end) { assert(pi->pi_offset == pi->pi_range.ar_end); if (at_end(&pi->pi_range)) { @@ -1409,7 +1413,7 @@ PyTypeObject _PyPositionsIterator = { sizeof(positionsiterator), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)positionsiter_dealloc, /* tp_dealloc */ + positionsiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1431,7 +1435,7 @@ PyTypeObject _PyPositionsIterator = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)positionsiter_next, /* tp_iternext */ + positionsiter_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1447,8 +1451,9 @@ PyTypeObject _PyPositionsIterator = { }; static PyObject* -code_positionsiterator(PyCodeObject* code, PyObject* Py_UNUSED(args)) +code_positionsiterator(PyObject *self, PyObject* Py_UNUSED(args)) { + PyCodeObject *code = (PyCodeObject*)self; positionsiterator* pi = (positionsiterator*)PyType_GenericAlloc(&_PyPositionsIterator, 0); if (pi == NULL) { return NULL; @@ -1875,16 +1880,18 @@ code_dealloc(PyCodeObject *co) #ifdef Py_GIL_DISABLED static int -code_traverse(PyCodeObject *co, visitproc visit, void *arg) +code_traverse(PyObject *self, visitproc visit, void *arg) { + PyCodeObject *co = (PyCodeObject*)self; Py_VISIT(co->co_consts); return 0; } #endif static PyObject * -code_repr(PyCodeObject *co) +code_repr(PyObject *self) { + PyCodeObject *co = (PyCodeObject*)self; int lineno; if (co->co_firstlineno != 0) lineno = co->co_firstlineno; @@ -1991,8 +1998,9 @@ code_richcompare(PyObject *self, PyObject *other, int op) } static Py_hash_t -code_hash(PyCodeObject *co) +code_hash(PyObject *self) { + PyCodeObject *co = (PyCodeObject*)self; Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ @@ -2053,8 +2061,9 @@ static PyMemberDef code_memberlist[] = { static PyObject * -code_getlnotab(PyCodeObject *code, void *closure) +code_getlnotab(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; if (PyErr_WarnEx(PyExc_DeprecationWarning, "co_lnotab is deprecated, use co_lines instead.", 1) < 0) { @@ -2064,51 +2073,57 @@ code_getlnotab(PyCodeObject *code, void *closure) } static PyObject * -code_getvarnames(PyCodeObject *code, void *closure) +code_getvarnames(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetVarnames(code); } static PyObject * -code_getcellvars(PyCodeObject *code, void *closure) +code_getcellvars(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetCellvars(code); } static PyObject * -code_getfreevars(PyCodeObject *code, void *closure) +code_getfreevars(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetFreevars(code); } static PyObject * -code_getcodeadaptive(PyCodeObject *code, void *closure) +code_getcodeadaptive(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return PyBytes_FromStringAndSize(code->co_code_adaptive, _PyCode_NBYTES(code)); } static PyObject * -code_getcode(PyCodeObject *code, void *closure) +code_getcode(PyObject *self, void *closure) { + PyCodeObject *code = (PyCodeObject*)self; return _PyCode_GetCode(code); } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, - {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL}, + {"co_lnotab", code_getlnotab, NULL, NULL}, + {"_co_code_adaptive", code_getcodeadaptive, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_varnames", (getter)code_getvarnames, NULL, NULL}, - {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, - {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"co_code", (getter)code_getcode, NULL, NULL}, + {"co_varnames", code_getvarnames, NULL, NULL}, + {"co_cellvars", code_getcellvars, NULL, NULL}, + {"co_freevars", code_getfreevars, NULL, NULL}, + {"co_code", code_getcode, NULL, NULL}, {0} }; static PyObject * -code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) +code_sizeof(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *co = (PyCodeObject*)self; size_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { @@ -2119,8 +2134,9 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) } static PyObject * -code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) +code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *code = (PyCodeObject*)self; return (PyObject *)new_linesiterator(code); } @@ -2262,9 +2278,9 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg) /* XXX code objects need to participate in GC? */ static struct PyMethodDef code_methods[] = { - {"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS}, - {"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS}, - {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS}, + {"__sizeof__", code_sizeof, METH_NOARGS}, + {"co_lines", code_linesiterator, METH_NOARGS}, + {"co_positions", code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS, @@ -2283,11 +2299,11 @@ PyTypeObject PyCode_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)code_repr, /* tp_repr */ + code_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)code_hash, /* tp_hash */ + code_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -2300,7 +2316,7 @@ PyTypeObject PyCode_Type = { #endif code_new__doc__, /* tp_doc */ #ifdef Py_GIL_DISABLED - (traverseproc)code_traverse, /* tp_traverse */ + code_traverse, /* tp_traverse */ #else 0, /* tp_traverse */ #endif @@ -2336,6 +2352,7 @@ _PyCode_ConstantKey(PyObject *op) if (op == Py_None || op == Py_Ellipsis || PyLong_CheckExact(op) || PyUnicode_CheckExact(op) + || PySlice_Check(op) /* code_richcompare() uses _PyCode_ConstantKey() internally */ || PyCode_Check(op)) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f38ab1b2865e99..b27599d2815c82 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -416,6 +416,8 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) +#define _Py_DICT_IMMORTAL_INITIAL_REFCNT PY_SSIZE_T_MIN + static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the @@ -428,7 +430,8 @@ static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); static inline void dictkeys_incref(PyDictKeysObject *dk) { - if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); return; } #ifdef Py_REF_DEBUG @@ -440,7 +443,8 @@ dictkeys_incref(PyDictKeysObject *dk) static inline void dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { - if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); return; } assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0); @@ -586,7 +590,7 @@ estimate_log2_keysize(Py_ssize_t n) * (which cannot fail and thus can do no allocation). */ static PyDictKeysObject empty_keys_struct = { - _Py_IMMORTAL_REFCNT, /* dk_refcnt */ + _Py_DICT_IMMORTAL_INITIAL_REFCNT, /* dk_refcnt */ 0, /* dk_log2_size */ 0, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ @@ -877,7 +881,7 @@ new_dict(PyInterpreterState *interp, mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(interp); + mp->_ma_watcher_tag = 0; ASSERT_CONSISTENT(mp); return (PyObject *)mp; } @@ -1488,7 +1492,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb *value_addr = value; if (value != NULL) { assert(ix >= 0); - Py_INCREF(value); + _Py_NewRefWithLock(value); } Py_END_CRITICAL_SECTION(); return ix; @@ -1678,8 +1682,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, } } - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); mp->ma_keys->dk_version = 0; Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); @@ -1698,7 +1701,6 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, STORE_VALUE(ep, value); STORE_HASH(ep, hash); } - mp->ma_version_tag = new_version; STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); assert(mp->ma_keys->dk_usable >= 0); @@ -1744,16 +1746,14 @@ insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, MAINTAIN_TRACKING(mp, key, value); PyObject *old_value = mp->ma_values->values[ix]; if (old_value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); STORE_USED(mp, mp->ma_used + 1); - mp->ma_version_tag = new_version; } else { - uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); - mp->ma_version_tag = new_version; // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_DECREF(old_value); @@ -1815,8 +1815,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_MODIFIED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); assert(!_PyDict_HasSplitTable(mp)); if (DK_IS_UNICODE(mp->ma_keys)) { @@ -1827,7 +1826,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; STORE_VALUE(ep, value); } - mp->ma_version_tag = new_version; } Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ ASSERT_CONSISTENT(mp); @@ -1857,8 +1855,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ assert(mp->ma_values == NULL); @@ -1879,7 +1876,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, STORE_VALUE(ep, value); } STORE_USED(mp, mp->ma_used + 1); - mp->ma_version_tag = new_version; newkeys->dk_usable--; newkeys->dk_nentries++; // We store the keys last so no one can see them in a partially inconsistent @@ -2612,7 +2608,7 @@ delete_index_from_values(PyDictValues *values, Py_ssize_t ix) static void delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, - PyObject *old_value, uint64_t new_version) + PyObject *old_value) { PyObject *old_key; @@ -2622,7 +2618,6 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, assert(hashpos >= 0); STORE_USED(mp, mp->ma_used - 1); - mp->ma_version_tag = new_version; if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); STORE_SPLIT_VALUE(mp, ix, NULL); @@ -2692,9 +2687,8 @@ delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) } PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); return 0; } @@ -2740,9 +2734,8 @@ delitemif_lock_held(PyObject *op, PyObject *key, if (res > 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); return 1; } else { return 0; @@ -2786,11 +2779,9 @@ clear_lock_held(PyObject *op) } /* Empty the dict... */ PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); // We don't inc ref empty keys because they're immortal ensure_shared_on_resize(mp); - mp->ma_version_tag = new_version; STORE_USED(mp, 0); if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); @@ -2950,9 +2941,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, assert(old_value != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, Py_NewRef(old_value)); ASSERT_CONSISTENT(mp); if (result) { @@ -3210,16 +3200,12 @@ static PyObject * dict_repr_lock_held(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; - Py_ssize_t i; PyObject *key = NULL, *value = NULL; - _PyUnicodeWriter writer; - int first; - ASSERT_DICT_LOCKED(mp); - i = Py_ReprEnter((PyObject *)mp); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("{...}") : NULL; + int res = Py_ReprEnter((PyObject *)mp); + if (res != 0) { + return (res > 0 ? PyUnicode_FromString("{...}") : NULL); } if (mp->ma_used == 0) { @@ -3227,66 +3213,70 @@ dict_repr_lock_held(PyObject *self) return PyUnicode_FromString("{}"); } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - /* "{" + "1: 2" + ", 3: 4" * (len - 1) + "}" */ - writer.min_length = 1 + 4 + (2 + 4) * (mp->ma_used - 1) + 1; + // "{" + "1: 2" + ", 3: 4" * (len - 1) + "}" + Py_ssize_t prealloc = 1 + 4 + 6 * (mp->ma_used - 1) + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; + } - if (_PyUnicodeWriter_WriteChar(&writer, '{') < 0) + if (PyUnicodeWriter_WriteChar(writer, '{') < 0) { goto error; + } /* Do repr() on each key+value pair, and insert ": " between them. Note that repr may mutate the dict. */ - i = 0; - first = 1; + Py_ssize_t i = 0; + int first = 1; while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { - PyObject *s; - int res; - - /* Prevent repr from deleting key or value during key format. */ + // Prevent repr from deleting key or value during key format. Py_INCREF(key); Py_INCREF(value); if (!first) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + // Write ", " + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { + goto error; + } } first = 0; - s = PyObject_Repr(key); - if (s == NULL) - goto error; - res = _PyUnicodeWriter_WriteStr(&writer, s); - Py_DECREF(s); - if (res < 0) + // Write repr(key) + if (PyUnicodeWriter_WriteRepr(writer, key) < 0) { goto error; + } - if (_PyUnicodeWriter_WriteASCIIString(&writer, ": ", 2) < 0) + // Write ": " + if (PyUnicodeWriter_WriteChar(writer, ':') < 0) { goto error; - - s = PyObject_Repr(value); - if (s == NULL) + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; - res = _PyUnicodeWriter_WriteStr(&writer, s); - Py_DECREF(s); - if (res < 0) + } + + // Write repr(value) + if (PyUnicodeWriter_WriteRepr(writer, value) < 0) { goto error; + } Py_CLEAR(key); Py_CLEAR(value); } - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteChar(&writer, '}') < 0) + if (PyUnicodeWriter_WriteChar(writer, '}') < 0) { goto error; + } Py_ReprLeave((PyObject *)mp); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: Py_ReprLeave((PyObject *)mp); - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_XDECREF(key); Py_XDECREF(value); return NULL; @@ -3717,8 +3707,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) ) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) return -1; @@ -3727,7 +3716,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; STORE_USED(mp, other->ma_used); - mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { @@ -3930,13 +3918,13 @@ dict_copy_impl(PyDictObject *self) } /* Copies the values, but does not change the reference - * counts of the objects in the array. */ + * counts of the objects in the array. + * Return NULL, but does *not* set an exception on failure */ static PyDictValues * copy_values(PyDictValues *values) { PyDictValues *newvalues = new_values(values->capacity); if (newvalues == NULL) { - PyErr_NoMemory(); return NULL; } newvalues->size = values->size; @@ -3982,7 +3970,7 @@ copy_lock_held(PyObject *o) split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); + split_copy->_ma_watcher_tag = 0; dictkeys_incref(mp->ma_keys); if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); @@ -4430,7 +4418,6 @@ dict_popitem_impl(PyDictObject *self) { Py_ssize_t i, j; PyObject *res; - uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); ASSERT_DICT_LOCKED(self); @@ -4473,8 +4460,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -4489,8 +4475,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -4508,7 +4493,6 @@ dict_popitem_impl(PyDictObject *self) /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ STORE_KEYS_NENTRIES(self->ma_keys, i); STORE_USED(self, self->ma_used - 1); - self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; } @@ -4759,8 +4743,7 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION( - _PyInterpreterState_GET()); + d->_ma_watcher_tag = 0; dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -7015,7 +6998,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr // Still no dict, we can read from the values assert(values->valid); value = values->values[ix]; - *attr = Py_XNewRef(value); + *attr = _Py_XNewRefWithLock(value); success = true; } @@ -7035,7 +7018,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); - *attr = Py_XNewRef(value); + *attr = _Py_XNewRefWithLock(value); success = true; } else { // Caller needs to lookup from the dictionary @@ -7216,6 +7199,13 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) PyDictValues *values = copy_values(mp->ma_values); if (values == NULL) { + /* Out of memory. Clear the dict */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = mp->ma_keys; + set_keys(mp, Py_EMPTY_KEYS); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + STORE_USED(mp, 0); + PyErr_NoMemory(); return -1; } mp->ma_values = values; @@ -7377,7 +7367,7 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag |= (1LL << watcher_id); return 0; } @@ -7392,7 +7382,7 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag &= ~(1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag &= ~(1LL << watcher_id); return 0; } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index fda62f159c1540..6fbe0f197eaebf 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -154,7 +154,7 @@ BaseException_str(PyBaseExceptionObject *self) { switch (PyTuple_GET_SIZE(self->args)) { case 0: - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); case 1: return PyObject_Str(PyTuple_GET_ITEM(self->args, 0)); default: @@ -2994,46 +2994,55 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeEncodeError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason and encoding as strings, which they might not be if they've been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; - encoding_str = PyObject_Str(uself->encoding); - if (encoding_str == NULL) + } + encoding_str = PyObject_Str(exc->encoding); + if (encoding_str == NULL) { goto done; + } + + Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object); + Py_ssize_t start = exc->start, end = exc->end; - if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { - Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start); const char *fmt; - if (badchar <= 0xff) + if (badchar <= 0xff) { fmt = "'%U' codec can't encode character '\\x%02x' in position %zd: %U"; - else if (badchar <= 0xffff) + } + else if (badchar <= 0xffff) { fmt = "'%U' codec can't encode character '\\u%04x' in position %zd: %U"; - else + } + else { fmt = "'%U' codec can't encode character '\\U%08x' in position %zd: %U"; + } result = PyUnicode_FromFormat( fmt, encoding_str, (int)badchar, - uself->start, + start, reason_str); } else { result = PyUnicode_FromFormat( "'%U' codec can't encode characters in position %zd-%zd: %U", encoding_str, - uself->start, - uself->end-1, + start, + end - 1, reason_str); } done: @@ -3107,41 +3116,46 @@ UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * UnicodeDecodeError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason and encoding as strings, which they might not be if they've been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; - encoding_str = PyObject_Str(uself->encoding); - if (encoding_str == NULL) + } + encoding_str = PyObject_Str(exc->encoding); + if (encoding_str == NULL) { goto done; + } + + Py_ssize_t len = PyBytes_GET_SIZE(exc->object); + Py_ssize_t start = exc->start, end = exc->end; - if (uself->start < PyBytes_GET_SIZE(uself->object) && uself->end == uself->start+1) { - int byte = (int)(PyBytes_AS_STRING(((PyUnicodeErrorObject *)self)->object)[uself->start]&0xff); + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + int badbyte = (int)(PyBytes_AS_STRING(exc->object)[start] & 0xff); result = PyUnicode_FromFormat( "'%U' codec can't decode byte 0x%02x in position %zd: %U", encoding_str, - byte, - uself->start, + badbyte, + start, reason_str); } else { result = PyUnicode_FromFormat( "'%U' codec can't decode bytes in position %zd-%zd: %U", encoding_str, - uself->start, - uself->end-1, - reason_str - ); + start, + end - 1, + reason_str); } done: Py_XDECREF(reason_str); @@ -3204,42 +3218,49 @@ UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, static PyObject * UnicodeTranslateError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; PyObject *result = NULL; PyObject *reason_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason as a string, which it might not be if it's been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; + } + + Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object); + Py_ssize_t start = exc->start, end = exc->end; - if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { - Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start); const char *fmt; - if (badchar <= 0xff) + if (badchar <= 0xff) { fmt = "can't translate character '\\x%02x' in position %zd: %U"; - else if (badchar <= 0xffff) + } + else if (badchar <= 0xffff) { fmt = "can't translate character '\\u%04x' in position %zd: %U"; - else + } + else { fmt = "can't translate character '\\U%08x' in position %zd: %U"; + } result = PyUnicode_FromFormat( fmt, (int)badchar, - uself->start, - reason_str - ); - } else { + start, + reason_str); + } + else { result = PyUnicode_FromFormat( "can't translate characters in position %zd-%zd: %U", - uself->start, - uself->end-1, - reason_str - ); + start, + end - 1, + reason_str); } done: Py_XDECREF(reason_str); @@ -3387,8 +3408,9 @@ _PyErr_NoMemory(PyThreadState *tstate) } static void -MemoryError_dealloc(PyBaseExceptionObject *self) +MemoryError_dealloc(PyObject *obj) { + PyBaseExceptionObject *self = (PyBaseExceptionObject *)obj; _PyObject_GC_UNTRACK(self); BaseException_clear(self); @@ -3447,7 +3469,7 @@ PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, + 0, MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, diff --git a/Objects/floatobject.c b/Objects/floatobject.c index dc3d8a3e5d0f4b..d66863febe8c86 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -134,6 +134,41 @@ PyFloat_FromDouble(double fval) return (PyObject *) op; } +#ifdef Py_GIL_DISABLED + +PyObject *_PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value) +{ + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + return PyFloat_FromDouble(value); +} + +#else // Py_GIL_DISABLED + +PyObject *_PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value) +{ + PyObject *left_o = PyStackRef_AsPyObjectSteal(left); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + if (Py_REFCNT(left_o) == 1) { + ((PyFloatObject *)left_o)->ob_fval = value; + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + return left_o; + } + else if (Py_REFCNT(right_o) == 1) { + ((PyFloatObject *)right_o)->ob_fval = value; + _Py_DECREF_NO_DEALLOC(left_o); + return right_o; + } + else { + PyObject *result = PyFloat_FromDouble(value); + _Py_DECREF_NO_DEALLOC(left_o); + _Py_DECREF_NO_DEALLOC(right_o); + return result; + } +} + +#endif // Py_GIL_DISABLED + static PyObject * float_from_string_inner(const char *s, Py_ssize_t len, void *obj) { @@ -406,19 +441,16 @@ float_richcompare(PyObject *v, PyObject *w, int op) } /* The signs are the same. */ /* Convert w to a double if it fits. In particular, 0 fits. */ - uint64_t nbits64 = _PyLong_NumBits(w); - if (nbits64 > (unsigned int)DBL_MAX_EXP) { + int64_t nbits64 = _PyLong_NumBits(w); + assert(nbits64 >= 0); + assert(!PyErr_Occurred()); + if (nbits64 > DBL_MAX_EXP) { /* This Python integer is larger than any finite C double. * Replace with little doubles * that give the same outcome -- w is so large that * its magnitude must exceed the magnitude of any * finite float. */ - if (nbits64 == (uint64_t)-1 && PyErr_Occurred()) { - /* This Python integer is so large that uint64_t isn't - * big enough to hold the # of bits. */ - PyErr_Clear(); - } i = (double)vsign; assert(wsign != 0); j = wsign * 2.0; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b567327f970836..f3a66ffc9aac8f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -310,14 +310,31 @@ framelocalsproxy_dealloc(PyObject *self) static PyObject * framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + if (PyTuple_GET_SIZE(args) != 1) { + PyErr_Format(PyExc_TypeError, + "FrameLocalsProxy expected 1 argument, got %zd", + PyTuple_GET_SIZE(args)); + return NULL; + } + PyObject *item = PyTuple_GET_ITEM(args, 0); + + if (!PyFrame_Check(item)) { + PyErr_Format(PyExc_TypeError, "expect frame, not %T", item); + return NULL; + } + PyFrameObject *frame = (PyFrameObject*)item; + + if (kwds != NULL && PyDict_Size(kwds) != 0) { + PyErr_SetString(PyExc_TypeError, + "FrameLocalsProxy takes no keyword arguments"); + return 0; + } + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); if (self == NULL) { return NULL; } - PyFrameObject *frame = (PyFrameObject*)PyTuple_GET_ITEM(args, 0); - assert(PyFrame_Check(frame)); - ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); return (PyObject *)self; @@ -1634,7 +1651,7 @@ frame_dealloc(PyFrameObject *f) /* Kill all local variables including specials, if we own them */ if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { PyStackRef_CLEAR(frame->f_executable); - Py_CLEAR(frame->f_funcobj); + PyStackRef_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); _PyStackRef *sp = frame->stackpointer; @@ -1790,7 +1807,7 @@ static void init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; - _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), + _PyFrame_Initialize(frame, PyStackRef_FromPyObjectNew(func), Py_XNewRef(locals), code, 0, NULL); } @@ -1861,14 +1878,15 @@ frame_init_get_vars(_PyInterpreterFrame *frame) PyCodeObject *co = _PyFrame_GetCode(frame); int lasti = _PyInterpreterFrame_LASTI(frame); if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS - && PyFunction_Check(frame->f_funcobj))) + && PyStackRef_FunctionCheck(frame->f_funcobj))) { /* Free vars are initialized */ return; } /* Free vars have not been initialized -- Do that */ - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + PyFunctionObject *func = _PyFrame_GetFunction(frame); + PyObject *closure = func->func_closure; int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 8df0da800980a9..855d1a2eeca819 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -128,7 +128,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + op->func_version = FUNC_VERSION_UNSET; // NOTE: functions created via FrameConstructor do not use deferred // reference counting because they are typically not part of cycles // nor accessed by multiple threads. @@ -207,7 +207,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + op->func_version = FUNC_VERSION_UNSET; if ((code_obj->co_flags & CO_NESTED) == 0) { // Use deferred reference counting for top-level functions, but not // nested functions because they are more likely to capture variables, @@ -287,31 +287,59 @@ functions is running. */ +static inline struct _func_version_cache_item * +get_cache_item(PyInterpreterState *interp, uint32_t version) +{ + return interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); +} + void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { + assert(func->func_version == FUNC_VERSION_UNSET); + assert(version >= FUNC_VERSION_FIRST_VALID); + // This should only be called from MAKE_FUNCTION. No code is specialized + // based on the version, so we do not need to stop the world to set it. + func->func_version = version; #ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); - if (func->func_version != 0) { - struct _func_version_cache_item *slot = - interp->func_state.func_version_cache - + (func->func_version % FUNC_VERSION_CACHE_SIZE); - if (slot->func == func) { - slot->func = NULL; - // Leave slot->code alone, there may be use for it. - } - } + struct _func_version_cache_item *slot = get_cache_item(interp, version); + slot->func = func; + slot->code = func->func_code; #endif - func->func_version = version; +} + +static void +func_clear_version(PyInterpreterState *interp, PyFunctionObject *func) +{ + if (func->func_version < FUNC_VERSION_FIRST_VALID) { + // Version was never set or has already been cleared. + return; + } #ifndef Py_GIL_DISABLED - if (version != 0) { - struct _func_version_cache_item *slot = - interp->func_state.func_version_cache - + (version % FUNC_VERSION_CACHE_SIZE); - slot->func = func; - slot->code = func->func_code; + struct _func_version_cache_item *slot = + get_cache_item(interp, func->func_version); + if (slot->func == func) { + slot->func = NULL; + // Leave slot->code alone, there may be use for it. } #endif + func->func_version = FUNC_VERSION_CLEARED; +} + +// Called when any of the critical function attributes are changed +static void +_PyFunction_ClearVersion(PyFunctionObject *func) +{ + if (func->func_version < FUNC_VERSION_FIRST_VALID) { + // Version was never set or has already been cleared. + return; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + func_clear_version(interp, func); + _PyEval_StartTheWorld(interp); } void @@ -319,9 +347,7 @@ _PyFunction_ClearCodeByVersion(uint32_t version) { #ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _func_version_cache_item *slot = - interp->func_state.func_version_cache - + (version % FUNC_VERSION_CACHE_SIZE); + struct _func_version_cache_item *slot = get_cache_item(interp, version); if (slot->code) { assert(PyCode_Check(slot->code)); PyCodeObject *code = (PyCodeObject *)slot->code; @@ -340,9 +366,7 @@ _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) return NULL; #else PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _func_version_cache_item *slot = - interp->func_state.func_version_cache - + (version % FUNC_VERSION_CACHE_SIZE); + struct _func_version_cache_item *slot = get_cache_item(interp, version); if (slot->code) { assert(PyCode_Check(slot->code)); PyCodeObject *code = (PyCodeObject *)slot->code; @@ -431,7 +455,7 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, (PyFunctionObject *) op, defaults); - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_defaults, defaults); return 0; } @@ -440,7 +464,7 @@ void PyFunction_SetVectorcall(PyFunctionObject *func, vectorcallfunc vectorcall) { assert(func != NULL); - _PyFunction_SetVersion(func, 0); + _PyFunction_ClearVersion(func); func->vectorcall = vectorcall; } @@ -473,7 +497,7 @@ PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, (PyFunctionObject *) op, defaults); - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_kwdefaults, defaults); return 0; } @@ -506,7 +530,7 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) Py_TYPE(closure)->tp_name); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_closure, closure); return 0; } @@ -604,8 +628,9 @@ static PyMemberDef func_memberlist[] = { }; static PyObject * -func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_code(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__code__") < 0) { return NULL; } @@ -614,10 +639,9 @@ func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { - Py_ssize_t nclosure; - int nfree; + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.func_code or to set it to anything * other than a code object. */ @@ -632,9 +656,9 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) return -1; } - nfree = ((PyCodeObject *)value)->co_nfreevars; - nclosure = (op->func_closure == NULL ? 0 : - PyTuple_GET_SIZE(op->func_closure)); + int nfree = ((PyCodeObject *)value)->co_nfreevars; + Py_ssize_t nclosure = (op->func_closure == NULL ? 0 : + PyTuple_GET_SIZE(op->func_closure)); if (nclosure != nfree) { PyErr_Format(PyExc_ValueError, "%U() requires a code object with %zd free vars," @@ -658,20 +682,22 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) } handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } static PyObject * -func_get_name(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_name(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); return Py_NewRef(op->func_name); } static int -func_set_name(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.func_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -684,14 +710,16 @@ func_set_name(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -func_get_qualname(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); return Py_NewRef(op->func_qualname); } static int -func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -704,8 +732,9 @@ func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } static PyObject * -func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_defaults(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__defaults__") < 0) { return NULL; } @@ -716,10 +745,11 @@ func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_defaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Legal to del f.func_defaults. * Can only set func_defaults to NULL or a tuple. */ + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; if (value != NULL && !PyTuple_Check(value)) { @@ -738,14 +768,15 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_defaults, Py_XNewRef(value)); return 0; } static PyObject * -func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_kwdefaults(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__kwdefaults__") < 0) { return NULL; @@ -757,8 +788,9 @@ func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_kwdefaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; /* Legal to del f.func_kwdefaults. @@ -779,14 +811,15 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_kwdefaults, Py_XNewRef(value)); return 0; } static PyObject * -func_get_annotate(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (op->func_annotate == NULL) { Py_RETURN_NONE; } @@ -794,8 +827,9 @@ func_get_annotate(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_annotate(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__annotate__ cannot be deleted"); @@ -818,8 +852,9 @@ func_set_annotate(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } static PyObject * -func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (op->func_annotations == NULL && (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate))) { op->func_annotations = PyDict_New(); @@ -831,8 +866,9 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; /* Legal to del f.func_annotations. @@ -849,8 +885,9 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno } static PyObject * -func_get_type_params(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_type_params(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (op->func_typeparams == NULL) { return PyTuple_New(0); } @@ -860,10 +897,11 @@ func_get_type_params(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_type_params(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_type_params(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Not legal to del f.__type_params__ or to set it to anything * other than a tuple object. */ + PyFunctionObject *op = _PyFunction_CAST(self); if (value == NULL || !PyTuple_Check(value)) { PyErr_SetString(PyExc_TypeError, "__type_params__ must be set to a tuple"); @@ -885,19 +923,15 @@ _Py_set_function_type_params(PyThreadState *Py_UNUSED(ignored), PyObject *func, } static PyGetSetDef func_getsetlist[] = { - {"__code__", (getter)func_get_code, (setter)func_set_code}, - {"__defaults__", (getter)func_get_defaults, - (setter)func_set_defaults}, - {"__kwdefaults__", (getter)func_get_kwdefaults, - (setter)func_set_kwdefaults}, - {"__annotations__", (getter)func_get_annotations, - (setter)func_set_annotations}, - {"__annotate__", (getter)func_get_annotate, (setter)func_set_annotate}, + {"__code__", func_get_code, func_set_code}, + {"__defaults__", func_get_defaults, func_set_defaults}, + {"__kwdefaults__", func_get_kwdefaults, func_set_kwdefaults}, + {"__annotations__", func_get_annotations, func_set_annotations}, + {"__annotate__", func_get_annotate, func_set_annotate}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, - {"__name__", (getter)func_get_name, (setter)func_set_name}, - {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, - {"__type_params__", (getter)func_get_type_params, - (setter)func_set_type_params}, + {"__name__", func_get_name, func_set_name}, + {"__qualname__", func_get_qualname, func_set_qualname}, + {"__type_params__", func_get_type_params, func_set_type_params}, {NULL} /* Sentinel */ }; @@ -1017,9 +1051,10 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, } static int -func_clear(PyFunctionObject *op) +func_clear(PyObject *self) { - _PyFunction_SetVersion(op, 0); + PyFunctionObject *op = _PyFunction_CAST(self); + func_clear_version(_PyInterpreterState_GET(), op); Py_CLEAR(op->func_globals); Py_CLEAR(op->func_builtins); Py_CLEAR(op->func_module); @@ -1042,8 +1077,9 @@ func_clear(PyFunctionObject *op) } static void -func_dealloc(PyFunctionObject *op) +func_dealloc(PyObject *self) { + PyFunctionObject *op = _PyFunction_CAST(self); assert(Py_REFCNT(op) == 0); Py_SET_REFCNT(op, 1); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); @@ -1056,8 +1092,7 @@ func_dealloc(PyFunctionObject *op) if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } - _PyFunction_SetVersion(op, 0); - (void)func_clear(op); + (void)func_clear((PyObject*)op); // These aren't cleared by func_clear(). Py_DECREF(op->func_code); Py_DECREF(op->func_name); @@ -1066,15 +1101,17 @@ func_dealloc(PyFunctionObject *op) } static PyObject* -func_repr(PyFunctionObject *op) +func_repr(PyObject *self) { + PyFunctionObject *op = _PyFunction_CAST(self); return PyUnicode_FromFormat("", op->func_qualname, op); } static int -func_traverse(PyFunctionObject *f, visitproc visit, void *arg) +func_traverse(PyObject *self, visitproc visit, void *arg) { + PyFunctionObject *f = _PyFunction_CAST(self); Py_VISIT(f->func_code); Py_VISIT(f->func_globals); Py_VISIT(f->func_builtins); @@ -1107,12 +1144,12 @@ PyTypeObject PyFunction_Type = { "function", sizeof(PyFunctionObject), 0, - (destructor)func_dealloc, /* tp_dealloc */ + func_dealloc, /* tp_dealloc */ offsetof(PyFunctionObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)func_repr, /* tp_repr */ + func_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1126,8 +1163,8 @@ PyTypeObject PyFunction_Type = { Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */ func_new__doc__, /* tp_doc */ - (traverseproc)func_traverse, /* tp_traverse */ - (inquiry)func_clear, /* tp_clear */ + func_traverse, /* tp_traverse */ + func_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyFunctionObject, func_weakreflist), /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -1251,9 +1288,14 @@ typedef struct { PyObject *cm_dict; } classmethod; +#define _PyClassMethod_CAST(cm) \ + (assert(PyObject_TypeCheck((cm), &PyClassMethod_Type)), \ + _Py_CAST(classmethod*, cm)) + static void -cm_dealloc(classmethod *cm) +cm_dealloc(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); _PyObject_GC_UNTRACK((PyObject *)cm); Py_XDECREF(cm->cm_callable); Py_XDECREF(cm->cm_dict); @@ -1261,16 +1303,18 @@ cm_dealloc(classmethod *cm) } static int -cm_traverse(classmethod *cm, visitproc visit, void *arg) +cm_traverse(PyObject *self, visitproc visit, void *arg) { + classmethod *cm = _PyClassMethod_CAST(self); Py_VISIT(cm->cm_callable); Py_VISIT(cm->cm_dict); return 0; } static int -cm_clear(classmethod *cm) +cm_clear(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); Py_CLEAR(cm->cm_callable); Py_CLEAR(cm->cm_dict); return 0; @@ -1317,8 +1361,9 @@ static PyMemberDef cm_memberlist[] = { }; static PyObject * -cm_get___isabstractmethod__(classmethod *cm, void *closure) +cm_get___isabstractmethod__(PyObject *self, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); int res = _PyObject_IsAbstract(cm->cm_callable); if (res == -1) { return NULL; @@ -1330,42 +1375,46 @@ cm_get___isabstractmethod__(classmethod *cm, void *closure) } static PyObject * -cm_get___annotations__(classmethod *cm, void *closure) +cm_get___annotations__(PyObject *self, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotations__)); } static int -cm_set___annotations__(classmethod *cm, PyObject *value, void *closure) +cm_set___annotations__(PyObject *self, PyObject *value, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotations__), value, "classmethod"); } static PyObject * -cm_get___annotate__(classmethod *cm, void *closure) +cm_get___annotate__(PyObject *self, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotate__)); } static int -cm_set___annotate__(classmethod *cm, PyObject *value, void *closure) +cm_set___annotate__(PyObject *self, PyObject *value, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotate__), value, "classmethod"); } static PyGetSetDef cm_getsetlist[] = { - {"__isabstractmethod__", - (getter)cm_get___isabstractmethod__, NULL, NULL, NULL}, + {"__isabstractmethod__", cm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, - {"__annotations__", (getter)cm_get___annotations__, (setter)cm_set___annotations__, NULL, NULL}, - {"__annotate__", (getter)cm_get___annotate__, (setter)cm_set___annotate__, NULL, NULL}, + {"__annotations__", cm_get___annotations__, cm_set___annotations__, NULL, NULL}, + {"__annotate__", cm_get___annotate__, cm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; static PyObject* -cm_repr(classmethod *cm) +cm_repr(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); return PyUnicode_FromFormat("", cm->cm_callable); } @@ -1397,12 +1446,12 @@ PyTypeObject PyClassMethod_Type = { "classmethod", sizeof(classmethod), 0, - (destructor)cm_dealloc, /* tp_dealloc */ + cm_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)cm_repr, /* tp_repr */ + cm_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1414,14 +1463,14 @@ PyTypeObject PyClassMethod_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, classmethod_doc, /* tp_doc */ - (traverseproc)cm_traverse, /* tp_traverse */ - (inquiry)cm_clear, /* tp_clear */ + cm_traverse, /* tp_traverse */ + cm_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ - cm_memberlist, /* tp_members */ + cm_memberlist, /* tp_members */ cm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -1470,9 +1519,14 @@ typedef struct { PyObject *sm_dict; } staticmethod; +#define _PyStaticMethod_CAST(cm) \ + (assert(PyObject_TypeCheck((cm), &PyStaticMethod_Type)), \ + _Py_CAST(staticmethod*, cm)) + static void -sm_dealloc(staticmethod *sm) +sm_dealloc(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); _PyObject_GC_UNTRACK((PyObject *)sm); Py_XDECREF(sm->sm_callable); Py_XDECREF(sm->sm_dict); @@ -1480,16 +1534,18 @@ sm_dealloc(staticmethod *sm) } static int -sm_traverse(staticmethod *sm, visitproc visit, void *arg) +sm_traverse(PyObject *self, visitproc visit, void *arg) { + staticmethod *sm = _PyStaticMethod_CAST(self); Py_VISIT(sm->sm_callable); Py_VISIT(sm->sm_dict); return 0; } static int -sm_clear(staticmethod *sm) +sm_clear(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); Py_CLEAR(sm->sm_callable); Py_CLEAR(sm->sm_dict); return 0; @@ -1540,8 +1596,9 @@ static PyMemberDef sm_memberlist[] = { }; static PyObject * -sm_get___isabstractmethod__(staticmethod *sm, void *closure) +sm_get___isabstractmethod__(PyObject *self, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); int res = _PyObject_IsAbstract(sm->sm_callable); if (res == -1) { return NULL; @@ -1553,41 +1610,45 @@ sm_get___isabstractmethod__(staticmethod *sm, void *closure) } static PyObject * -sm_get___annotations__(staticmethod *sm, void *closure) +sm_get___annotations__(PyObject *self, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotations__)); } static int -sm_set___annotations__(staticmethod *sm, PyObject *value, void *closure) +sm_set___annotations__(PyObject *self, PyObject *value, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotations__), value, "staticmethod"); } static PyObject * -sm_get___annotate__(staticmethod *sm, void *closure) +sm_get___annotate__(PyObject *self, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotate__)); } static int -sm_set___annotate__(staticmethod *sm, PyObject *value, void *closure) +sm_set___annotate__(PyObject *self, PyObject *value, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotate__), value, "staticmethod"); } static PyGetSetDef sm_getsetlist[] = { - {"__isabstractmethod__", - (getter)sm_get___isabstractmethod__, NULL, NULL, NULL}, + {"__isabstractmethod__", sm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, - {"__annotations__", (getter)sm_get___annotations__, (setter)sm_set___annotations__, NULL, NULL}, - {"__annotate__", (getter)sm_get___annotate__, (setter)sm_set___annotate__, NULL, NULL}, + {"__annotations__", sm_get___annotations__, sm_set___annotations__, NULL, NULL}, + {"__annotate__", sm_get___annotate__, sm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; static PyObject* -sm_repr(staticmethod *sm) +sm_repr(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); return PyUnicode_FromFormat("", sm->sm_callable); } @@ -1617,12 +1678,12 @@ PyTypeObject PyStaticMethod_Type = { "staticmethod", sizeof(staticmethod), 0, - (destructor)sm_dealloc, /* tp_dealloc */ + sm_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)sm_repr, /* tp_repr */ + sm_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1634,14 +1695,14 @@ PyTypeObject PyStaticMethod_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, staticmethod_doc, /* tp_doc */ - (traverseproc)sm_traverse, /* tp_traverse */ - (inquiry)sm_clear, /* tp_clear */ + sm_traverse, /* tp_traverse */ + sm_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ - sm_memberlist, /* tp_members */ + sm_memberlist, /* tp_members */ sm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ diff --git a/Objects/genobject.c b/Objects/genobject.c index 5dc8f926557b52..19c2c4e3331a89 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -17,9 +17,20 @@ #include "pystats.h" -static PyObject *gen_close(PyGenObject *, PyObject *); -static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); -static PyObject *async_gen_athrow_new(PyAsyncGenObject *, PyObject *); +// Forward declarations +static PyObject* gen_close(PyObject *, PyObject *); +static PyObject* async_gen_asend_new(PyAsyncGenObject *, PyObject *); +static PyObject* async_gen_athrow_new(PyAsyncGenObject *, PyObject *); + + +#define _PyGen_CAST(op) \ + _Py_CAST(PyGenObject*, (op)) +#define _PyCoroObject_CAST(op) \ + (assert(PyCoro_CheckExact(op)), \ + _Py_CAST(PyCoroObject*, (op))) +#define _PyAsyncGenObject_CAST(op) \ + _Py_CAST(PyAsyncGenObject*, (op)) + static const char *NON_INIT_CORO_MSG = "can't send non-None value to a " "just-started coroutine"; @@ -42,8 +53,9 @@ PyGen_GetCode(PyGenObject *gen) { } static int -gen_traverse(PyGenObject *gen, visitproc visit, void *arg) +gen_traverse(PyObject *self, visitproc visit, void *arg) { + PyGenObject *gen = _PyGen_CAST(self); Py_VISIT(gen->gi_name); Py_VISIT(gen->gi_qualname); if (gen->gi_frame_state != FRAME_CLEARED) { @@ -58,10 +70,7 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) else { // We still need to visit the code object when the frame is cleared to // ensure that it's kept alive if the reference is deferred. - int err = _PyGC_VisitStackRef(&gen->gi_iframe.f_executable, visit, arg); - if (err) { - return err; - } + _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ @@ -110,7 +119,7 @@ _PyGen_Finalize(PyObject *self) _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } else { - PyObject *res = gen_close(gen, NULL); + PyObject *res = gen_close((PyObject*)gen, NULL); if (res == NULL) { if (PyErr_Occurred()) { PyErr_WriteUnraisable(self); @@ -126,9 +135,9 @@ _PyGen_Finalize(PyObject *self) } static void -gen_dealloc(PyGenObject *gen) +gen_dealloc(PyObject *self) { - PyObject *self = (PyObject *) gen; + PyGenObject *gen = _PyGen_CAST(self); _PyObject_GC_UNTRACK(gen); @@ -265,8 +274,9 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, } static PySendResult -PyGen_am_send(PyGenObject *gen, PyObject *arg, PyObject **result) +PyGen_am_send(PyObject *self, PyObject *arg, PyObject **result) { + PyGenObject *gen = _PyGen_CAST(self); return gen_send_ex2(gen, arg, result, 0, 0); } @@ -295,9 +305,9 @@ PyDoc_STRVAR(send_doc, return next yielded value or raise StopIteration."); static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +gen_send(PyObject *gen, PyObject *arg) { - return gen_send_ex(gen, arg, 0, 0); + return gen_send_ex((PyGenObject*)gen, arg, 0, 0); } PyDoc_STRVAR(close_doc, @@ -314,7 +324,7 @@ gen_close_iter(PyObject *yf) PyObject *retval = NULL; if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) { - retval = gen_close((PyGenObject *)yf, NULL); + retval = gen_close((PyObject *)yf, NULL); if (retval == NULL) return -1; } @@ -359,11 +369,9 @@ _PyGen_yf(PyGenObject *gen) } static PyObject * -gen_close(PyGenObject *gen, PyObject *args) +gen_close(PyObject *self, PyObject *args) { - PyObject *retval; - int err = 0; - + PyGenObject *gen = _PyGen_CAST(self); if (gen->gi_frame_state == FRAME_CREATED) { gen->gi_frame_state = FRAME_COMPLETED; @@ -372,7 +380,9 @@ gen_close(PyGenObject *gen, PyObject *args) if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } + PyObject *yf = _PyGen_yf(gen); + int err = 0; if (yf) { PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; @@ -397,7 +407,8 @@ gen_close(PyGenObject *gen, PyObject *args) if (err == 0) { PyErr_SetNone(PyExc_GeneratorExit); } - retval = gen_send_ex(gen, Py_None, 1, 1); + + PyObject *retval = gen_send_ex(gen, Py_None, 1, 1); if (retval) { const char *msg = "generator ignored GeneratorExit"; if (PyCoro_CheckExact(gen)) { @@ -410,10 +421,12 @@ gen_close(PyGenObject *gen, PyObject *args) return NULL; } assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { PyErr_Clear(); /* ignore this error */ Py_RETURN_NONE; } + /* if the generator returned a value while closing, StopIteration was * raised in gen_send_ex() above; retrieve and return the value here */ if (_PyGen_FetchStopIterationValue(&retval) == 0) { @@ -590,10 +603,12 @@ gen_throw(PyGenObject *gen, PyObject *const *args, Py_ssize_t nargs) static PyObject * -gen_iternext(PyGenObject *gen) +gen_iternext(PyObject *self) { + assert(PyGen_CheckExact(self) || PyCoro_CheckExact(self)); + PyGenObject *gen = _PyGen_CAST(self); + PyObject *result; - assert(PyGen_CheckExact(gen) || PyCoro_CheckExact(gen)); if (gen_send_ex2(gen, NULL, &result, 0, 0) == PYGEN_RETURN) { if (result != Py_None) { _PyGen_SetStopIterationValue(result); @@ -667,21 +682,24 @@ _PyGen_FetchStopIterationValue(PyObject **pvalue) } static PyObject * -gen_repr(PyGenObject *gen) +gen_repr(PyObject *self) { + PyGenObject *gen = _PyGen_CAST(self); return PyUnicode_FromFormat("", gen->gi_qualname, gen); } static PyObject * -gen_get_name(PyGenObject *op, void *Py_UNUSED(ignored)) +gen_get_name(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); return Py_NewRef(op->gi_name); } static int -gen_set_name(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) +gen_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); /* Not legal to del gen.gi_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -694,14 +712,16 @@ gen_set_name(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_get_qualname(PyGenObject *op, void *Py_UNUSED(ignored)) +gen_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); return Py_NewRef(op->gi_qualname); } static int -gen_set_qualname(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) +gen_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); /* Not legal to del gen.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -714,18 +734,20 @@ gen_set_qualname(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_getyieldfrom(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getyieldfrom(PyObject *gen, void *Py_UNUSED(ignored)) { - PyObject *yf = _PyGen_yf(gen); - if (yf == NULL) + PyObject *yf = _PyGen_yf(_PyGen_CAST(gen)); + if (yf == NULL) { Py_RETURN_NONE; + } return yf; } static PyObject * -gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); if (gen->gi_frame_state == FRAME_EXECUTING) { Py_RETURN_TRUE; } @@ -733,8 +755,9 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) } static PyObject * -gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return PyBool_FromLong(FRAME_STATE_SUSPENDED(gen->gi_frame_state)); } @@ -751,8 +774,9 @@ _gen_getframe(PyGenObject *gen, const char *const name) } static PyObject * -gen_getframe(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getframe(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return _gen_getframe(gen, "gi_frame"); } @@ -766,22 +790,23 @@ _gen_getcode(PyGenObject *gen, const char *const name) } static PyObject * -gen_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getcode(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return _gen_getcode(gen, "gi_code"); } static PyGetSetDef gen_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the generator")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the generator")}, - {"gi_yieldfrom", (getter)gen_getyieldfrom, NULL, + {"gi_yieldfrom", gen_getyieldfrom, NULL, PyDoc_STR("object being iterated by yield from, or None")}, - {"gi_running", (getter)gen_getrunning, NULL, NULL}, - {"gi_frame", (getter)gen_getframe, NULL, NULL}, - {"gi_suspended", (getter)gen_getsuspended, NULL, NULL}, - {"gi_code", (getter)gen_getcode, NULL, NULL}, + {"gi_running", gen_getrunning, NULL, NULL}, + {"gi_frame", gen_getframe, NULL, NULL}, + {"gi_suspended", gen_getsuspended, NULL, NULL}, + {"gi_code", gen_getcode, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -803,9 +828,9 @@ PyDoc_STRVAR(sizeof__doc__, "gen.__sizeof__() -> size of gen in memory, in bytes"); static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, - {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, - {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, + {"send", gen_send, METH_O, send_doc}, + {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, + {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ @@ -815,7 +840,7 @@ static PyAsyncMethods gen_as_async = { 0, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + PyGen_am_send, /* am_send */ }; @@ -825,12 +850,12 @@ PyTypeObject PyGen_Type = { offsetof(PyGenObject, gi_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &gen_as_async, /* tp_as_async */ - (reprfunc)gen_repr, /* tp_repr */ + gen_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -842,12 +867,12 @@ PyTypeObject PyGen_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gen_traverse, /* tp_traverse */ + gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyGenObject, gi_weakreflist), /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)gen_iternext, /* tp_iternext */ + gen_iternext, /* tp_iternext */ gen_methods, /* tp_methods */ gen_memberlist, /* tp_members */ gen_getsetlist, /* tp_getset */ @@ -906,17 +931,18 @@ _Py_MakeCoro(PyFunctionObject *func) return make_gen(&PyGen_Type, func); } if (coro_flags == CO_ASYNC_GENERATOR) { - PyAsyncGenObject *o; - o = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); - if (o == NULL) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + if (ag == NULL) { return NULL; } - o->ag_origin_or_finalizer = NULL; - o->ag_closed = 0; - o->ag_hooks_inited = 0; - o->ag_running_async = 0; - return (PyObject*)o; + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; } + assert (coro_flags == CO_COROUTINE); PyObject *coro = make_gen(&PyCoro_Type, func); if (!coro) { @@ -998,6 +1024,11 @@ typedef struct { PyCoroObject *cw_coroutine; } PyCoroWrapper; +#define _PyCoroWrapper_CAST(op) \ + (assert(Py_IS_TYPE((op), &_PyCoroWrapper_Type)), \ + _Py_CAST(PyCoroWrapper*, (op))) + + static int gen_is_coroutine(PyObject *o) { @@ -1060,14 +1091,15 @@ _PyCoro_GetAwaitableIter(PyObject *o) } static PyObject * -coro_repr(PyCoroObject *coro) +coro_repr(PyObject *self) { + PyCoroObject *coro = _PyCoroObject_CAST(self); return PyUnicode_FromFormat("", coro->cr_qualname, coro); } static PyObject * -coro_await(PyCoroObject *coro) +coro_await(PyObject *coro) { PyCoroWrapper *cw = PyObject_GC_New(PyCoroWrapper, &_PyCoroWrapper_Type); if (cw == NULL) { @@ -1079,7 +1111,7 @@ coro_await(PyCoroObject *coro) } static PyObject * -coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) +coro_get_cr_await(PyObject *coro, void *Py_UNUSED(ignored)) { PyObject *yf = _PyGen_yf((PyGenObject *) coro); if (yf == NULL) @@ -1088,8 +1120,9 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyCoroObject *coro = _PyCoroObject_CAST(self); if (FRAME_STATE_SUSPENDED(coro->cr_frame_state)) { Py_RETURN_TRUE; } @@ -1097,8 +1130,9 @@ cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { + PyCoroObject *coro = _PyCoroObject_CAST(self); if (coro->cr_frame_state == FRAME_EXECUTING) { Py_RETURN_TRUE; } @@ -1106,29 +1140,29 @@ cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getframe(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getframe(PyObject *coro, void *Py_UNUSED(ignored)) { - return _gen_getframe((PyGenObject *)coro, "cr_frame"); + return _gen_getframe(_PyGen_CAST(coro), "cr_frame"); } static PyObject * -cr_getcode(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getcode(PyObject *coro, void *Py_UNUSED(ignored)) { - return _gen_getcode((PyGenObject *)coro, "cr_code"); + return _gen_getcode(_PyGen_CAST(coro), "cr_code"); } static PyGetSetDef coro_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the coroutine")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the coroutine")}, - {"cr_await", (getter)coro_get_cr_await, NULL, + {"cr_await", coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, - {"cr_running", (getter)cr_getrunning, NULL, NULL}, - {"cr_frame", (getter)cr_getframe, NULL, NULL}, - {"cr_code", (getter)cr_getcode, NULL, NULL}, - {"cr_suspended", (getter)cr_getsuspended, NULL, NULL}, + {"cr_running", cr_getrunning, NULL, NULL}, + {"cr_frame", cr_getframe, NULL, NULL}, + {"cr_code", cr_getcode, NULL, NULL}, + {"cr_suspended", cr_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1155,19 +1189,19 @@ PyDoc_STRVAR(coro_close_doc, "close() -> raise GeneratorExit inside coroutine."); static PyMethodDef coro_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, coro_send_doc}, + {"send", gen_send, METH_O, coro_send_doc}, {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, - {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, + {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; static PyAsyncMethods coro_as_async = { - (unaryfunc)coro_await, /* am_await */ + coro_await, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyCoro_Type = { @@ -1176,12 +1210,12 @@ PyTypeObject PyCoro_Type = { offsetof(PyCoroObject, cr_iframe.localsplus),/* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &coro_as_async, /* tp_as_async */ - (reprfunc)coro_repr, /* tp_repr */ + coro_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1193,7 +1227,7 @@ PyTypeObject PyCoro_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gen_traverse, /* tp_traverse */ + gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyCoroObject, cr_weakreflist), /* tp_weaklistoffset */ @@ -1223,49 +1257,55 @@ PyTypeObject PyCoro_Type = { }; static void -coro_wrapper_dealloc(PyCoroWrapper *cw) +coro_wrapper_dealloc(PyObject *self) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); _PyObject_GC_UNTRACK((PyObject *)cw); Py_CLEAR(cw->cw_coroutine); PyObject_GC_Del(cw); } static PyObject * -coro_wrapper_iternext(PyCoroWrapper *cw) +coro_wrapper_iternext(PyObject *self) { - return gen_iternext((PyGenObject *)cw->cw_coroutine); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_iternext((PyObject *)cw->cw_coroutine); } static PyObject * -coro_wrapper_send(PyCoroWrapper *cw, PyObject *arg) +coro_wrapper_send(PyObject *self, PyObject *arg) { - return gen_send((PyGenObject *)cw->cw_coroutine, arg); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_send((PyObject *)cw->cw_coroutine, arg); } static PyObject * -coro_wrapper_throw(PyCoroWrapper *cw, PyObject *const *args, Py_ssize_t nargs) +coro_wrapper_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); return gen_throw((PyGenObject *)cw->cw_coroutine, args, nargs); } static PyObject * -coro_wrapper_close(PyCoroWrapper *cw, PyObject *args) +coro_wrapper_close(PyObject *self, PyObject *args) { - return gen_close((PyGenObject *)cw->cw_coroutine, args); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_close((PyObject *)cw->cw_coroutine, args); } static int -coro_wrapper_traverse(PyCoroWrapper *cw, visitproc visit, void *arg) +coro_wrapper_traverse(PyObject *self, visitproc visit, void *arg) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); Py_VISIT((PyObject *)cw->cw_coroutine); return 0; } static PyMethodDef coro_wrapper_methods[] = { - {"send",(PyCFunction)coro_wrapper_send, METH_O, coro_send_doc}, - {"throw",_PyCFunction_CAST(coro_wrapper_throw), - METH_FASTCALL, coro_throw_doc}, - {"close",(PyCFunction)coro_wrapper_close, METH_NOARGS, coro_close_doc}, + {"send", coro_wrapper_send, METH_O, coro_send_doc}, + {"throw", _PyCFunction_CAST(coro_wrapper_throw), METH_FASTCALL, + coro_throw_doc}, + {"close", coro_wrapper_close, METH_NOARGS, coro_close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1274,7 +1314,7 @@ PyTypeObject _PyCoroWrapper_Type = { "coroutine_wrapper", sizeof(PyCoroWrapper), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)coro_wrapper_dealloc, /* destructor tp_dealloc */ + coro_wrapper_dealloc, /* destructor tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1291,12 +1331,12 @@ PyTypeObject _PyCoroWrapper_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "A wrapper object implementing __await__ for coroutines.", - (traverseproc)coro_wrapper_traverse, /* tp_traverse */ + coro_wrapper_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)coro_wrapper_iternext, /* tp_iternext */ + coro_wrapper_iternext, /* tp_iternext */ coro_wrapper_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1390,6 +1430,9 @@ typedef struct PyAsyncGenASend { AwaitableState ags_state; } PyAsyncGenASend; +#define _PyAsyncGenASend_CAST(op) \ + _Py_CAST(PyAsyncGenASend*, (op)) + typedef struct PyAsyncGenAThrow { PyObject_HEAD @@ -1411,19 +1454,24 @@ typedef struct _PyAsyncGenWrappedValue { #define _PyAsyncGenWrappedValue_CheckExact(o) \ Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type) +#define _PyAsyncGenWrappedValue_CAST(op) \ + (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ + _Py_CAST(_PyAsyncGenWrappedValue*, (op))) static int -async_gen_traverse(PyAsyncGenObject *gen, visitproc visit, void *arg) +async_gen_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(gen->ag_origin_or_finalizer); - return gen_traverse((PyGenObject*)gen, visit, arg); + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); + Py_VISIT(ag->ag_origin_or_finalizer); + return gen_traverse((PyObject*)ag, visit, arg); } static PyObject * -async_gen_repr(PyAsyncGenObject *o) +async_gen_repr(PyObject *self) { + PyAsyncGenObject *o = _PyAsyncGenObject_CAST(self); return PyUnicode_FromFormat("", o->ag_qualname, o); } @@ -1467,12 +1515,13 @@ async_gen_init_hooks(PyAsyncGenObject *o) static PyObject * -async_gen_anext(PyAsyncGenObject *o) +async_gen_anext(PyObject *self) { - if (async_gen_init_hooks(o)) { + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); + if (async_gen_init_hooks(ag)) { return NULL; } - return async_gen_asend_new(o, NULL); + return async_gen_asend_new(ag, NULL); } @@ -1513,20 +1562,21 @@ async_gen_athrow(PyAsyncGenObject *o, PyObject *args) } static PyObject * -ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +ag_getframe(PyObject *ag, void *Py_UNUSED(ignored)) { return _gen_getframe((PyGenObject *)ag, "ag_frame"); } static PyObject * -ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +ag_getcode(PyObject *gen, void *Py_UNUSED(ignored)) { - return _gen_getcode(gen, "ag_code"); + return _gen_getcode((PyGenObject*)gen, "ag_code"); } static PyObject * -ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +ag_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); if (FRAME_STATE_SUSPENDED(ag->ag_frame_state)) { Py_RETURN_TRUE; } @@ -1534,15 +1584,15 @@ ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) } static PyGetSetDef async_gen_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the async generator")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the async generator")}, - {"ag_await", (getter)coro_get_cr_await, NULL, + {"ag_await", coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, - {"ag_frame", (getter)ag_getframe, NULL, NULL}, - {"ag_code", (getter)ag_getcode, NULL, NULL}, - {"ag_suspended", (getter)ag_getsuspended, NULL, NULL}, + {"ag_frame", ag_getframe, NULL, NULL}, + {"ag_code", ag_getcode, NULL, NULL}, + {"ag_suspended", ag_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1580,8 +1630,8 @@ static PyMethodDef async_gen_methods[] = { static PyAsyncMethods async_gen_as_async = { 0, /* am_await */ PyObject_SelfIter, /* am_aiter */ - (unaryfunc)async_gen_anext, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + async_gen_anext, /* am_anext */ + PyGen_am_send, /* am_send */ }; @@ -1591,12 +1641,12 @@ PyTypeObject PyAsyncGen_Type = { offsetof(PyAsyncGenObject, ag_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &async_gen_as_async, /* tp_as_async */ - (reprfunc)async_gen_repr, /* tp_repr */ + async_gen_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1608,7 +1658,7 @@ PyTypeObject PyAsyncGen_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_traverse, /* tp_traverse */ + async_gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyAsyncGenObject, ag_weakreflist), /* tp_weaklistoffset */ @@ -1641,17 +1691,18 @@ PyTypeObject PyAsyncGen_Type = { PyObject * PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) { - PyAsyncGenObject *o; - o = (PyAsyncGenObject *)gen_new_with_qualname( - &PyAsyncGen_Type, f, name, qualname); - if (o == NULL) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)gen_new_with_qualname(&PyAsyncGen_Type, f, + name, qualname); + if (ag == NULL) { return NULL; } - o->ag_origin_or_finalizer = NULL; - o->ag_closed = 0; - o->ag_hooks_inited = 0; - o->ag_running_async = 0; - return (PyObject*)o; + + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; } static PyObject * @@ -1688,36 +1739,38 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) static void -async_gen_asend_dealloc(PyAsyncGenASend *o) +async_gen_asend_dealloc(PyObject *self) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)o)) { + assert(PyAsyncGenASend_CheckExact(self)); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + + if (PyObject_CallFinalizerFromDealloc(self)) { return; } - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->ags_gen); - Py_CLEAR(o->ags_sendval); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(ags->ags_gen); + Py_CLEAR(ags->ags_sendval); - assert(PyAsyncGenASend_CheckExact(o)); - _PyGC_CLEAR_FINALIZED((PyObject *)o); + _PyGC_CLEAR_FINALIZED(self); - _Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del); + _Py_FREELIST_FREE(async_gen_asends, self, PyObject_GC_Del); } static int -async_gen_asend_traverse(PyAsyncGenASend *o, visitproc visit, void *arg) +async_gen_asend_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->ags_gen); - Py_VISIT(o->ags_sendval); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + Py_VISIT(ags->ags_gen); + Py_VISIT(ags->ags_sendval); return 0; } static PyObject * -async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) +async_gen_asend_send(PyObject *self, PyObject *arg) { - PyObject *result; - + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( PyExc_RuntimeError, @@ -1741,7 +1794,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) } o->ags_gen->ag_running_async = 1; - result = gen_send((PyGenObject*)o->ags_gen, arg); + PyObject *result = gen_send((PyObject*)o->ags_gen, arg); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { @@ -1753,16 +1806,16 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) static PyObject * -async_gen_asend_iternext(PyAsyncGenASend *o) +async_gen_asend_iternext(PyObject *ags) { - return async_gen_asend_send(o, NULL); + return async_gen_asend_send(ags, NULL); } static PyObject * -async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t nargs) +async_gen_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *result; + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( @@ -1784,7 +1837,7 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg o->ags_gen->ag_running_async = 1; } - result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); + PyObject *result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { @@ -1797,13 +1850,14 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg static PyObject * -async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) +async_gen_asend_close(PyObject *self, PyObject *args) { - PyObject *result; + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { Py_RETURN_NONE; } - result = async_gen_asend_throw(o, &PyExc_GeneratorExit, 1); + + PyObject *result = async_gen_asend_throw(self, &PyExc_GeneratorExit, 1); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_StopIteration) || PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || @@ -1813,25 +1867,26 @@ async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) Py_RETURN_NONE; } return result; - } else { - Py_DECREF(result); - PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); - return NULL; } + + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; } static void -async_gen_asend_finalize(PyAsyncGenASend *o) +async_gen_asend_finalize(PyObject *self) { - if (o->ags_state == AWAITABLE_STATE_INIT) { - _PyErr_WarnUnawaitedAgenMethod(o->ags_gen, &_Py_ID(asend)); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + if (ags->ags_state == AWAITABLE_STATE_INIT) { + _PyErr_WarnUnawaitedAgenMethod(ags->ags_gen, &_Py_ID(asend)); } } static PyMethodDef async_gen_asend_methods[] = { - {"send", (PyCFunction)async_gen_asend_send, METH_O, send_doc}, + {"send", async_gen_asend_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_asend_throw), METH_FASTCALL, throw_doc}, - {"close", (PyCFunction)async_gen_asend_close, METH_NOARGS, close_doc}, + {"close", async_gen_asend_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1850,7 +1905,7 @@ PyTypeObject _PyAsyncGenASend_Type = { sizeof(PyAsyncGenASend), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_asend_dealloc, /* tp_dealloc */ + async_gen_asend_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1867,12 +1922,12 @@ PyTypeObject _PyAsyncGenASend_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_asend_traverse, /* tp_traverse */ + async_gen_asend_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)async_gen_asend_iternext, /* tp_iternext */ + async_gen_asend_iternext, /* tp_iternext */ async_gen_asend_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1884,29 +1939,27 @@ PyTypeObject _PyAsyncGenASend_Type = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - .tp_finalize = (destructor)async_gen_asend_finalize, + .tp_finalize = async_gen_asend_finalize, }; static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { - PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends); - if (o == NULL) { - o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); - if (o == NULL) { + PyAsyncGenASend *ags = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends); + if (ags == NULL) { + ags = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); + if (ags == NULL) { return NULL; } } - o->ags_gen = (PyAsyncGenObject*)Py_NewRef(gen); - - o->ags_sendval = Py_XNewRef(sendval); + ags->ags_gen = (PyAsyncGenObject*)Py_NewRef(gen); + ags->ags_sendval = Py_XNewRef(sendval); + ags->ags_state = AWAITABLE_STATE_INIT; - o->ags_state = AWAITABLE_STATE_INIT; - - _PyObject_GC_TRACK((PyObject*)o); - return (PyObject*)o; + _PyObject_GC_TRACK((PyObject*)ags); + return (PyObject*)ags; } @@ -1914,19 +1967,20 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) static void -async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) +async_gen_wrapped_val_dealloc(PyObject *self) { - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->agw_val); - _Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del); + _PyAsyncGenWrappedValue *agw = _PyAsyncGenWrappedValue_CAST(self); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(agw->agw_val); + _Py_FREELIST_FREE(async_gens, self, PyObject_GC_Del); } static int -async_gen_wrapped_val_traverse(_PyAsyncGenWrappedValue *o, - visitproc visit, void *arg) +async_gen_wrapped_val_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->agw_val); + _PyAsyncGenWrappedValue *agw = _PyAsyncGenWrappedValue_CAST(self); + Py_VISIT(agw->agw_val); return 0; } @@ -1937,7 +1991,7 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { sizeof(_PyAsyncGenWrappedValue), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_wrapped_val_dealloc, /* tp_dealloc */ + async_gen_wrapped_val_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1954,7 +2008,7 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_wrapped_val_traverse, /* tp_traverse */ + async_gen_wrapped_val_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -1996,34 +2050,40 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) /* ---------- Async Generator AThrow awaitable ------------ */ +#define _PyAsyncGenAThrow_CAST(op) \ + (assert(Py_IS_TYPE((op), &_PyAsyncGenAThrow_Type)), \ + _Py_CAST(PyAsyncGenAThrow*, (op))) static void -async_gen_athrow_dealloc(PyAsyncGenAThrow *o) +async_gen_athrow_dealloc(PyObject *self) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)o)) { + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + if (PyObject_CallFinalizerFromDealloc(self)) { return; } - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->agt_gen); - Py_CLEAR(o->agt_args); - PyObject_GC_Del(o); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(agt->agt_gen); + Py_CLEAR(agt->agt_args); + PyObject_GC_Del(self); } static int -async_gen_athrow_traverse(PyAsyncGenAThrow *o, visitproc visit, void *arg) +async_gen_athrow_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->agt_gen); - Py_VISIT(o->agt_args); + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + Py_VISIT(agt->agt_gen); + Py_VISIT(agt->agt_args); return 0; } static PyObject * -async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) +async_gen_athrow_send(PyObject *self, PyObject *arg) { - PyGenObject *gen = (PyGenObject*)o->agt_gen; + PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); + PyGenObject *gen = _PyGen_CAST(o->agt_gen); PyObject *retval; if (o->agt_state == AWAITABLE_STATE_CLOSED) { @@ -2106,7 +2166,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) assert(o->agt_state == AWAITABLE_STATE_ITER); - retval = gen_send((PyGenObject *)gen, arg); + retval = gen_send((PyObject *)gen, arg); if (o->agt_args) { return async_gen_unwrap_value(o->agt_gen, retval); } else { @@ -2153,9 +2213,9 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) static PyObject * -async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t nargs) +async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *retval; + PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); if (o->agt_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( @@ -2184,7 +2244,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na o->agt_gen->ag_running_async = 1; } - retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); + PyObject *retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { retval = async_gen_unwrap_value(o->agt_gen, retval); if (retval == NULL) { @@ -2192,7 +2252,8 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na o->agt_state = AWAITABLE_STATE_CLOSED; } return retval; - } else { + } + else { /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { o->agt_gen->ag_running_async = 0; @@ -2222,20 +2283,21 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na static PyObject * -async_gen_athrow_iternext(PyAsyncGenAThrow *o) +async_gen_athrow_iternext(PyObject *agt) { - return async_gen_athrow_send(o, Py_None); + return async_gen_athrow_send(agt, Py_None); } static PyObject * -async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) +async_gen_athrow_close(PyObject *self, PyObject *args) { - PyObject *result; - if (o->agt_state == AWAITABLE_STATE_CLOSED) { + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + if (agt->agt_state == AWAITABLE_STATE_CLOSED) { Py_RETURN_NONE; } - result = async_gen_athrow_throw(o, &PyExc_GeneratorExit, 1); + PyObject *result = async_gen_athrow_throw((PyObject*)agt, + &PyExc_GeneratorExit, 1); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_StopIteration) || PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || @@ -2263,10 +2325,10 @@ async_gen_athrow_finalize(PyAsyncGenAThrow *o) } static PyMethodDef async_gen_athrow_methods[] = { - {"send", (PyCFunction)async_gen_athrow_send, METH_O, send_doc}, + {"send", async_gen_athrow_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_athrow_throw), METH_FASTCALL, throw_doc}, - {"close", (PyCFunction)async_gen_athrow_close, METH_NOARGS, close_doc}, + {"close", async_gen_athrow_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -2285,7 +2347,7 @@ PyTypeObject _PyAsyncGenAThrow_Type = { sizeof(PyAsyncGenAThrow), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_athrow_dealloc, /* tp_dealloc */ + async_gen_athrow_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -2302,12 +2364,12 @@ PyTypeObject _PyAsyncGenAThrow_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_athrow_traverse, /* tp_traverse */ + async_gen_athrow_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)async_gen_athrow_iternext, /* tp_iternext */ + async_gen_athrow_iternext, /* tp_iternext */ async_gen_athrow_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ diff --git a/Objects/listobject.c b/Objects/listobject.c index 8abe15d6674140..930aefde325a7c 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -522,49 +522,48 @@ list_dealloc(PyObject *self) static PyObject * list_repr_impl(PyListObject *v) { - PyObject *s; - _PyUnicodeWriter writer; - Py_ssize_t i = Py_ReprEnter((PyObject*)v); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("[...]") : NULL; + int res = Py_ReprEnter((PyObject*)v); + if (res != 0) { + return (res > 0 ? PyUnicode_FromString("[...]") : NULL); } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; /* "[" + "1" + ", 2" * (len - 1) + "]" */ - writer.min_length = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + Py_ssize_t prealloc = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; + } - if (_PyUnicodeWriter_WriteChar(&writer, '[') < 0) + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { goto error; + } /* Do repr() on each element. Note that this may mutate the list, so must refetch the list size on each iteration. */ - for (i = 0; i < Py_SIZE(v); ++i) { + for (Py_ssize_t i = 0; i < Py_SIZE(v); ++i) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; + } } - s = PyObject_Repr(v->ob_item[i]); - if (s == NULL) - goto error; - - if (_PyUnicodeWriter_WriteStr(&writer, s) < 0) { - Py_DECREF(s); + if (PyUnicodeWriter_WriteRepr(writer, v->ob_item[i]) < 0) { goto error; } - Py_DECREF(s); } - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteChar(&writer, ']') < 0) + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { goto error; + } Py_ReprLeave((PyObject *)v); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_ReprLeave((PyObject *)v); return NULL; } diff --git a/Objects/longobject.c b/Objects/longobject.c index d34c8b6d71ab3f..4e948940485730 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -31,6 +31,13 @@ class int "PyObject *" "&PyLong_Type" /* If defined, use algorithms from the _pylong.py module */ #define WITH_PYLONG_MODULE 1 +// Forward declarations +static PyLongObject* long_neg(PyLongObject *v); +static PyLongObject *x_divrem(PyLongObject *, PyLongObject *, PyLongObject **); +static PyObject* long_long(PyObject *v); +static PyObject* long_lshift_int64(PyLongObject *a, int64_t shiftby); + + static inline void _Py_DECREF_INT(PyLongObject *op) { @@ -133,8 +140,16 @@ long_normalize(PyLongObject *v) /* Allocate a new int object with size digits. Return NULL and set exception if we run out of memory. */ -#define MAX_LONG_DIGITS \ +#if SIZEOF_SIZE_T < 8 +# define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, long_value.ob_digit))/sizeof(digit)) +#else +/* Guarantee that the number of bits fits in int64_t. + This is more than an exbibyte, that is more than many of modern + architectures support in principle. + -1 is added to avoid overflow in _PyLong_Frexp(). */ +# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) +#endif PyLongObject * _PyLong_New(Py_ssize_t size) @@ -258,17 +273,17 @@ _PyLong_FromLarge(stwodigits ival) } /* Create a new int object from a C word-sized int */ -static inline PyObject * +static inline PyLongObject * _PyLong_FromSTwoDigits(stwodigits x) { if (IS_SMALL_INT(x)) { - return get_small_int((sdigit)x); + return (PyLongObject*)get_small_int((sdigit)x); } assert(x != 0); if (is_medium_int(x)) { - return _PyLong_FromMedium((sdigit)x); + return (PyLongObject*)_PyLong_FromMedium((sdigit)x); } - return _PyLong_FromLarge(x); + return (PyLongObject*)_PyLong_FromLarge(x); } /* If a freshly-allocated int is already shared, it must @@ -284,7 +299,7 @@ _PyLong_Negate(PyLongObject **x_p) return; } - *x_p = (PyLongObject *)_PyLong_FromSTwoDigits(-medium_value(x)); + *x_p = _PyLong_FromSTwoDigits(-medium_value(x)); Py_DECREF(x); } @@ -804,11 +819,11 @@ bit_length_digit(digit x) return _Py_bit_length((unsigned long)x); } -uint64_t +int64_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; - uint64_t result = 0; + int64_t result = 0; Py_ssize_t ndigits; int msd_bits; @@ -818,21 +833,12 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->long_value.ob_digit[ndigits - 1]; - if ((uint64_t)(ndigits - 1) > UINT64_MAX / (uint64_t)PyLong_SHIFT) - goto Overflow; - result = (uint64_t)(ndigits - 1) * (uint64_t)PyLong_SHIFT; + assert(ndigits <= INT64_MAX / PyLong_SHIFT); + result = (int64_t)(ndigits - 1) * PyLong_SHIFT; msd_bits = bit_length_digit(msd); - if (UINT64_MAX - msd_bits < result) - goto Overflow; result += msd_bits; } return result; - - Overflow: - /* Very unlikely. Such integer would require more than 2 exbibytes of RAM. */ - PyErr_SetString(PyExc_OverflowError, "int has too many bits " - "to express in a 64-bit integer"); - return (uint64_t)-1; } PyObject * @@ -1247,15 +1253,12 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) /* Calculates the number of bits required for the *absolute* value * of v. This does not take sign into account, only magnitude. */ - uint64_t nb = _PyLong_NumBits((PyObject *)v); - if (nb == (uint64_t)-1) { - res = -1; - } else { - /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up - * multiples of 8 to the next byte, but we add an implied bit for - * the sign and it cancels out. */ - res = (Py_ssize_t)(nb / 8) + 1; - } + int64_t nb = _PyLong_NumBits((PyObject *)v); + assert(nb >= 0); + /* Normally this would be ((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; /* Two edge cases exist that are best handled after extracting the * bits. These may result in us reporting overflow when the value @@ -2623,8 +2626,6 @@ long_from_binary_base(const char *start, const char *end, Py_ssize_t digits, int return 0; } -static PyObject *long_neg(PyLongObject *v); - #ifdef WITH_PYLONG_MODULE /* asymptotically faster str-to-long conversion for base 10, using _pylong.py */ static int @@ -3153,11 +3154,6 @@ PyLong_FromUnicodeObject(PyObject *u, int base) return NULL; } -/* forward */ -static PyLongObject *x_divrem - (PyLongObject *, PyLongObject *, PyLongObject **); -static PyObject *long_long(PyObject *v); - /* Int division with remainder, top-level routine */ static int @@ -3415,7 +3411,8 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) double _PyLong_Frexp(PyLongObject *a, int64_t *e) { - Py_ssize_t a_size, shift_digits, shift_bits, x_size; + Py_ssize_t a_size, shift_digits, x_size; + int shift_bits; int64_t a_bits; /* See below for why x_digits is always large enough. */ digit rem; @@ -3432,14 +3429,7 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) *e = 0; return 0.0; } - int msd_bits = bit_length_digit(a->long_value.ob_digit[a_size-1]); - /* The following is an overflow-free version of the check - "if ((a_size - 1) * PyLong_SHIFT + msd_bits > PY_SSIZE_T_MAX) ..." */ - if (a_size >= (INT64_MAX - 1) / PyLong_SHIFT + 1 && - (a_size > (INT64_MAX - 1) / PyLong_SHIFT + 1 || - msd_bits > (INT64_MAX - 1) % PyLong_SHIFT + 1)) - goto overflow; - a_bits = (int64_t)(a_size - 1) * PyLong_SHIFT + msd_bits; + a_bits = _PyLong_NumBits((PyObject *)a); /* Shift the first DBL_MANT_DIG + 2 bits of a into x_digits[0:x_size] (shifting left if a_bits <= DBL_MANT_DIG + 2). @@ -3468,18 +3458,18 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) */ if (a_bits <= DBL_MANT_DIG + 2) { shift_digits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) / PyLong_SHIFT; - shift_bits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) % PyLong_SHIFT; + shift_bits = (DBL_MANT_DIG + 2 - (int)a_bits) % PyLong_SHIFT; x_size = shift_digits; rem = v_lshift(x_digits + x_size, a->long_value.ob_digit, a_size, - (int)shift_bits); + shift_bits); x_size += a_size; x_digits[x_size++] = rem; } else { shift_digits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT); - shift_bits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); + shift_bits = (int)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); rem = v_rshift(x_digits, a->long_value.ob_digit + shift_digits, - a_size - shift_digits, (int)shift_bits); + a_size - shift_digits, shift_bits); x_size = a_size - shift_digits; /* For correct rounding below, we need the least significant bit of x to be 'sticky' for this shift: if any of the bits @@ -3505,21 +3495,13 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) /* Rescale; make correction if result is 1.0. */ dx /= 4.0 * EXP2_DBL_MANT_DIG; if (dx == 1.0) { - if (a_bits == INT64_MAX) - goto overflow; + assert(a_bits < INT64_MAX); dx = 0.5; a_bits += 1; } *e = a_bits; return _PyLong_IsNegative(a) ? -dx : dx; - - overflow: - /* exponent > PY_SSIZE_T_MAX */ - PyErr_SetString(PyExc_OverflowError, - "huge integer: number of bits overflows a Py_ssize_t"); - *e = 0; - return -1.0; } /* Get a C double from an int object. Rounds to the nearest double, @@ -3547,7 +3529,9 @@ PyLong_AsDouble(PyObject *v) return (double)medium_value((PyLongObject *)v); } x = _PyLong_Frexp((PyLongObject *)v, &exponent); - if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { + assert(exponent >= 0); + assert(!PyErr_Occurred()); + if (exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, "int too large to convert to float"); return -1.0; @@ -3762,11 +3746,12 @@ x_sub(PyLongObject *a, PyLongObject *b) return maybe_small_long(long_normalize(z)); } -PyObject * -_PyLong_Add(PyLongObject *a, PyLongObject *b) +static PyLongObject * +long_add(PyLongObject *a, PyLongObject *b) { if (_PyLong_BothAreCompact(a, b)) { - return _PyLong_FromSTwoDigits(medium_value(a) + medium_value(b)); + stwodigits z = medium_value(a) + medium_value(b); + return _PyLong_FromSTwoDigits(z); } PyLongObject *z; @@ -3791,24 +3776,31 @@ _PyLong_Add(PyLongObject *a, PyLongObject *b) else z = x_add(a, b); } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Add(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_add(a, b); } static PyObject * -long_add(PyLongObject *a, PyLongObject *b) +long_add_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Add(a, b); + return (PyObject*)long_add((PyLongObject*)a, (PyLongObject*)b); } -PyObject * -_PyLong_Subtract(PyLongObject *a, PyLongObject *b) -{ - PyLongObject *z; +static PyLongObject * +long_sub(PyLongObject *a, PyLongObject *b) +{ if (_PyLong_BothAreCompact(a, b)) { return _PyLong_FromSTwoDigits(medium_value(a) - medium_value(b)); } + + PyLongObject *z; if (_PyLong_IsNegative(a)) { if (_PyLong_IsNegative(b)) { z = x_sub(b, a); @@ -3827,16 +3819,23 @@ _PyLong_Subtract(PyLongObject *a, PyLongObject *b) else z = x_sub(a, b); } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Subtract(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_sub(a, b); } static PyObject * -long_sub(PyLongObject *a, PyLongObject *b) +long_sub_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Subtract(a, b); + return (PyObject*)long_sub((PyLongObject*)a, (PyLongObject*)b); } + /* Grade school multiplication, ignoring the signs. * Returns the absolute value of the product, or NULL if error. */ @@ -4252,32 +4251,35 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) return NULL; } -PyObject * -_PyLong_Multiply(PyLongObject *a, PyLongObject *b) -{ - PyLongObject *z; +static PyLongObject* +long_mul(PyLongObject *a, PyLongObject *b) +{ /* fast path for single-digit multiplication */ if (_PyLong_BothAreCompact(a, b)) { stwodigits v = medium_value(a) * medium_value(b); return _PyLong_FromSTwoDigits(v); } - z = k_mul(a, b); + PyLongObject *z = k_mul(a, b); /* Negate if exactly one of the inputs is negative. */ if (!_PyLong_SameSign(a, b) && z) { _PyLong_Negate(&z); - if (z == NULL) - return NULL; } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Multiply(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_mul(a, b); } static PyObject * -long_mul(PyLongObject *a, PyLongObject *b) +long_mul_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Multiply(a, b); + return (PyObject*)long_mul((PyLongObject*)a, (PyLongObject*)b); } /* Fast modulo division for single-digit longs. */ @@ -4432,13 +4434,13 @@ l_divmod(PyLongObject *v, PyLongObject *w, if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || (_PyLong_IsPositive(mod) && _PyLong_IsNegative(w))) { PyLongObject *temp; - temp = (PyLongObject *) long_add(mod, w); + temp = long_add(mod, w); Py_SETREF(mod, temp); if (mod == NULL) { Py_DECREF(div); return -1; } - temp = (PyLongObject *) long_sub(div, (PyLongObject *)_PyLong_GetOne()); + temp = long_sub(div, (PyLongObject *)_PyLong_GetOne()); if (temp == NULL) { Py_DECREF(mod); Py_DECREF(div); @@ -4479,7 +4481,7 @@ l_mod(PyLongObject *v, PyLongObject *w, PyLongObject **pmod) if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || (_PyLong_IsPositive(mod) && _PyLong_IsNegative(w))) { PyLongObject *temp; - temp = (PyLongObject *) long_add(mod, w); + temp = long_add(mod, w); Py_SETREF(mod, temp); if (mod == NULL) return -1; @@ -4826,21 +4828,12 @@ long_divmod(PyObject *a, PyObject *b) static PyLongObject * long_invmod(PyLongObject *a, PyLongObject *n) { - PyLongObject *b, *c; - /* Should only ever be called for positive n */ assert(_PyLong_IsPositive(n)); - b = (PyLongObject *)PyLong_FromLong(1L); - if (b == NULL) { - return NULL; - } - c = (PyLongObject *)PyLong_FromLong(0L); - if (c == NULL) { - Py_DECREF(b); - return NULL; - } Py_INCREF(a); + PyLongObject *b = (PyLongObject *)Py_NewRef(_PyLong_GetOne()); + PyLongObject *c = (PyLongObject *)Py_NewRef(_PyLong_GetZero()); Py_INCREF(n); /* references now owned: a, b, c, n */ @@ -4857,7 +4850,7 @@ long_invmod(PyLongObject *a, PyLongObject *n) if (t == NULL) { goto Error; } - s = (PyLongObject *)long_sub(b, t); + s = long_sub(b, t); Py_DECREF(t); if (s == NULL) { goto Error; @@ -5152,7 +5145,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } if (negativeOutput && !_PyLong_IsZero(z)) { - temp = (PyLongObject *)long_sub(z, c); + temp = long_sub(z, c); if (temp == NULL) goto Error; Py_SETREF(z, temp); @@ -5175,13 +5168,15 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } static PyObject * -long_invert(PyLongObject *v) +long_invert(PyObject *self) { + PyLongObject *v = _PyLong_CAST(self); + /* Implement ~x as -(x+1) */ - PyLongObject *x; if (_PyLong_IsCompact(v)) - return _PyLong_FromSTwoDigits(~medium_value(v)); - x = (PyLongObject *) long_add(v, (PyLongObject *)_PyLong_GetOne()); + return (PyObject*)_PyLong_FromSTwoDigits(~medium_value(v)); + + PyLongObject *x = long_add(v, (PyLongObject *)_PyLong_GetOne()); if (x == NULL) return NULL; _PyLong_Negate(&x); @@ -5190,64 +5185,45 @@ long_invert(PyLongObject *v) return (PyObject *)x; } -static PyObject * +static PyLongObject * long_neg(PyLongObject *v) { - PyLongObject *z; - if (_PyLong_IsCompact(v)) + if (_PyLong_IsCompact(v)) { return _PyLong_FromSTwoDigits(-medium_value(v)); - z = (PyLongObject *)_PyLong_Copy(v); - if (z != NULL) + } + + PyLongObject *z = (PyLongObject *)_PyLong_Copy(v); + if (z != NULL) { _PyLong_FlipSign(z); - return (PyObject *)z; + } + return z; } static PyObject * +long_neg_method(PyObject *v) +{ + return (PyObject*)long_neg(_PyLong_CAST(v)); +} + +static PyLongObject* long_abs(PyLongObject *v) { if (_PyLong_IsNegative(v)) return long_neg(v); else - return long_long((PyObject *)v); + return (PyLongObject*)long_long((PyObject *)v); } -static int -long_bool(PyLongObject *v) +static PyObject * +long_abs_method(PyObject *v) { - return !_PyLong_IsZero(v); + return (PyObject*)long_abs(_PyLong_CAST(v)); } -/* wordshift, remshift = divmod(shiftby, PyLong_SHIFT) */ static int -divmod_shift(PyObject *shiftby, Py_ssize_t *wordshift, digit *remshift) -{ - assert(PyLong_Check(shiftby)); - assert(!_PyLong_IsNegative((PyLongObject *)shiftby)); - Py_ssize_t lshiftby = PyLong_AsSsize_t((PyObject *)shiftby); - if (lshiftby >= 0) { - *wordshift = lshiftby / PyLong_SHIFT; - *remshift = lshiftby % PyLong_SHIFT; - return 0; - } - /* PyLong_Check(shiftby) is true and shiftby is not negative, so it must - be that PyLong_AsSsize_t raised an OverflowError. */ - assert(PyErr_ExceptionMatches(PyExc_OverflowError)); - PyErr_Clear(); - PyLongObject *wordshift_obj = divrem1((PyLongObject *)shiftby, PyLong_SHIFT, remshift); - if (wordshift_obj == NULL) { - return -1; - } - *wordshift = PyLong_AsSsize_t((PyObject *)wordshift_obj); - Py_DECREF(wordshift_obj); - if (*wordshift >= 0 && *wordshift < PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(digit)) { - return 0; - } - PyErr_Clear(); - /* Clip the value. With such large wordshift the right shift - returns 0 and the left shift raises an error in _PyLong_New(). */ - *wordshift = PY_SSIZE_T_MAX / sizeof(digit); - *remshift = 0; - return 0; +long_bool(PyObject *v) +{ + return !_PyLong_IsZero(_PyLong_CAST(v)); } /* Inner function for both long_rshift and _PyLong_Rshift, shifting an @@ -5273,7 +5249,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) m = medium_value(a); shift = wordshift == 0 ? remshift : PyLong_SHIFT; x = m < 0 ? ~(~m >> shift) : m >> shift; - return _PyLong_FromSTwoDigits(x); + return (PyObject*)_PyLong_FromSTwoDigits(x); } a_negative = _PyLong_IsNegative(a); @@ -5343,8 +5319,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_rshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5355,24 +5330,35 @@ long_rshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) - return NULL; - return long_rshift1((PyLongObject *)a, wordshift, remshift); + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + PyErr_Clear(); + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } + return _PyLong_Rshift(a, shiftby); } /* Return a >> shiftby. */ PyObject * -_PyLong_Rshift(PyObject *a, uint64_t shiftby) +_PyLong_Rshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { if (_PyLong_IsNegative((PyLongObject *)a)) { return PyLong_FromLong(-1); } @@ -5397,7 +5383,7 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) stwodigits m = medium_value(a); // bypass undefined shift operator behavior stwodigits x = m < 0 ? -(-m << remshift) : m << remshift; - return _PyLong_FromSTwoDigits(x); + return (PyObject*)_PyLong_FromSTwoDigits(x); } oldsize = _PyLong_DigitCount(a); @@ -5427,49 +5413,61 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) return (PyObject *) maybe_small_long(z); } + static PyObject * -long_lshift(PyObject *a, PyObject *b) +long_lshift_method(PyObject *aa, PyObject *bb) { - Py_ssize_t wordshift; - digit remshift; - - CHECK_BINOP(a, b); + CHECK_BINOP(aa, bb); + PyLongObject *a = (PyLongObject*)aa; + PyLongObject *b = (PyLongObject*)bb; - if (_PyLong_IsNegative((PyLongObject *)b)) { + if (_PyLong_IsNegative(b)) { PyErr_SetString(PyExc_ValueError, "negative shift count"); return NULL; } - if (_PyLong_IsZero((PyLongObject *)a)) { + if (_PyLong_IsZero(a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) + + int64_t shiftby; + if (PyLong_AsInt64(bb, &shiftby) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + } return NULL; - return long_lshift1((PyLongObject *)a, wordshift, remshift); + } + return long_lshift_int64(a, shiftby); } /* Return a << shiftby. */ -PyObject * -_PyLong_Lshift(PyObject *a, uint64_t shiftby) +static PyObject * +long_lshift_int64(PyLongObject *a, int64_t shiftby) { - Py_ssize_t wordshift; - digit remshift; + assert(shiftby >= 0); - assert(PyLong_Check(a)); - if (_PyLong_IsZero((PyLongObject *)a)) { + if (_PyLong_IsZero(a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); return NULL; } #endif - wordshift = (Py_ssize_t)(shiftby / PyLong_SHIFT); - remshift = (digit)(shiftby % PyLong_SHIFT); - return long_lshift1((PyLongObject *)a, wordshift, remshift); + Py_ssize_t wordshift = (Py_ssize_t)(shiftby / PyLong_SHIFT); + digit remshift = (digit)(shiftby % PyLong_SHIFT); + return long_lshift1(a, wordshift, remshift); +} + +PyObject * +_PyLong_Lshift(PyObject *a, int64_t shiftby) +{ + return long_lshift_int64(_PyLong_CAST(a), shiftby); } + /* Compute two's complement of digit vector a[0:m], writing result to z[0:m]. The digit vector a need not be normalized, but should not be entirely zero. a and z may point to the same digit vector. */ @@ -5617,7 +5615,7 @@ long_and(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); } return long_bitwise(x, '&', y); } @@ -5629,7 +5627,7 @@ long_xor(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) ^ medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) ^ medium_value(y)); } return long_bitwise(x, '^', y); } @@ -5641,7 +5639,7 @@ long_or(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) | medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) | medium_value(y)); } return long_bitwise(x, '|', y); } @@ -5675,10 +5673,10 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } /* Initial reduction: make sure that 0 <= b <= a. */ - a = (PyLongObject *)long_abs(a); + a = long_abs(a); if (a == NULL) return NULL; - b = (PyLongObject *)long_abs(b); + b = long_abs(b); if (b == NULL) { Py_DECREF(a); return NULL; @@ -6060,12 +6058,11 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /* compare twice the remainder with the divisor, to see if we need to adjust the quotient and remainder */ - PyObject *one = _PyLong_GetOne(); // borrowed reference - twice_rem = long_lshift((PyObject *)rem, one); + twice_rem = long_lshift_int64(rem, 1); if (twice_rem == NULL) goto error; if (quo_is_neg) { - temp = long_neg((PyLongObject*)twice_rem); + temp = (PyObject*)long_neg((PyLongObject*)twice_rem); Py_SETREF(twice_rem, temp); if (twice_rem == NULL) goto error; @@ -6076,18 +6073,19 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) quo_is_odd = (quo->long_value.ob_digit[0] & 1) != 0; if ((_PyLong_IsNegative((PyLongObject *)b) ? cmp < 0 : cmp > 0) || (cmp == 0 && quo_is_odd)) { /* fix up quotient */ + PyObject *one = _PyLong_GetOne(); // borrowed reference if (quo_is_neg) - temp = long_sub(quo, (PyLongObject *)one); + temp = (PyObject*)long_sub(quo, (PyLongObject *)one); else - temp = long_add(quo, (PyLongObject *)one); + temp = (PyObject*)long_add(quo, (PyLongObject *)one); Py_SETREF(quo, (PyLongObject *)temp); if (quo == NULL) goto error; /* and remainder */ if (quo_is_neg) - temp = long_add(rem, (PyLongObject *)b); + temp = (PyObject*)long_add(rem, (PyLongObject *)b); else - temp = long_sub(rem, (PyLongObject *)b); + temp = (PyObject*)long_sub(rem, (PyLongObject *)b); Py_SETREF(rem, (PyLongObject *)temp); if (rem == NULL) goto error; @@ -6123,8 +6121,6 @@ static PyObject * int___round___impl(PyObject *self, PyObject *o_ndigits) /*[clinic end generated code: output=954fda6b18875998 input=30c2aec788263144]*/ { - PyObject *temp, *result, *ndigits; - /* To round an integer m to the nearest 10**n (n positive), we make use of * the divmod_near operation, defined by: * @@ -6142,7 +6138,7 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) if (o_ndigits == Py_None) return long_long(self); - ndigits = _PyNumber_Index(o_ndigits); + PyObject *ndigits = _PyNumber_Index(o_ndigits); if (ndigits == NULL) return NULL; @@ -6153,12 +6149,12 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) } /* result = self - divmod_near(self, 10 ** -ndigits)[1] */ - temp = long_neg((PyLongObject*)ndigits); + PyObject *temp = (PyObject*)long_neg((PyLongObject*)ndigits); Py_SETREF(ndigits, temp); if (ndigits == NULL) return NULL; - result = PyLong_FromLong(10L); + PyObject *result = PyLong_FromLong(10); if (result == NULL) { Py_DECREF(ndigits); return NULL; @@ -6175,8 +6171,8 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) if (result == NULL) return NULL; - temp = long_sub((PyLongObject *)self, - (PyLongObject *)PyTuple_GET_ITEM(result, 1)); + temp = (PyObject*)long_sub((PyLongObject*)self, + (PyLongObject*)PyTuple_GET_ITEM(result, 1)); Py_SETREF(result, temp); return result; @@ -6213,11 +6209,10 @@ static PyObject * int_bit_length_impl(PyObject *self) /*[clinic end generated code: output=fc1977c9353d6a59 input=e4eb7a587e849a32]*/ { - uint64_t nbits = _PyLong_NumBits(self); - if (nbits == (uint64_t)-1) { - return NULL; - } - return PyLong_FromUnsignedLongLong(nbits); + int64_t nbits = _PyLong_NumBits(self); + assert(nbits >= 0); + assert(!PyErr_Occurred()); + return PyLong_FromInt64(nbits); } static int @@ -6251,40 +6246,13 @@ int_bit_count_impl(PyObject *self) PyLongObject *z = (PyLongObject *)self; Py_ssize_t ndigits = _PyLong_DigitCount(z); - Py_ssize_t bit_count = 0; + int64_t bit_count = 0; - /* Each digit has up to PyLong_SHIFT ones, so the accumulated bit count - from the first PY_SSIZE_T_MAX/PyLong_SHIFT digits can't overflow a - Py_ssize_t. */ - Py_ssize_t ndigits_fast = Py_MIN(ndigits, PY_SSIZE_T_MAX/PyLong_SHIFT); - for (Py_ssize_t i = 0; i < ndigits_fast; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { bit_count += popcount_digit(z->long_value.ob_digit[i]); } - PyObject *result = PyLong_FromSsize_t(bit_count); - if (result == NULL) { - return NULL; - } - - /* Use Python integers if bit_count would overflow. */ - for (Py_ssize_t i = ndigits_fast; i < ndigits; i++) { - PyObject *x = PyLong_FromLong(popcount_digit(z->long_value.ob_digit[i])); - if (x == NULL) { - goto error; - } - PyObject *y = long_add((PyLongObject *)result, (PyLongObject *)x); - Py_DECREF(x); - if (y == NULL) { - goto error; - } - Py_SETREF(result, y); - } - - return result; - - error: - Py_DECREF(result); - return NULL; + return PyLong_FromInt64(bit_count); } /*[clinic input] @@ -6537,18 +6505,18 @@ Base 0 means to interpret the base from the string as an integer literal.\n\ 4"); static PyNumberMethods long_as_number = { - (binaryfunc)long_add, /*nb_add*/ - (binaryfunc)long_sub, /*nb_subtract*/ - (binaryfunc)long_mul, /*nb_multiply*/ + long_add_method, /*nb_add*/ + long_sub_method, /*nb_subtract*/ + long_mul_method, /*nb_multiply*/ long_mod, /*nb_remainder*/ long_divmod, /*nb_divmod*/ long_pow, /*nb_power*/ - (unaryfunc)long_neg, /*nb_negative*/ + long_neg_method, /*nb_negative*/ long_long, /*tp_positive*/ - (unaryfunc)long_abs, /*tp_absolute*/ - (inquiry)long_bool, /*tp_bool*/ - (unaryfunc)long_invert, /*nb_invert*/ - long_lshift, /*nb_lshift*/ + long_abs_method, /*tp_absolute*/ + long_bool, /*tp_bool*/ + long_invert, /*nb_invert*/ + long_lshift_method, /*nb_lshift*/ long_rshift, /*nb_rshift*/ long_and, /*nb_and*/ long_xor, /*nb_xor*/ diff --git a/Objects/methodobject.c b/Objects/methodobject.c index d6773a264101dc..345da4607423cf 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -156,8 +156,9 @@ PyCMethod_GetClass(PyObject *op) /* Methods (the standard built-in methods, that is) */ static void -meth_dealloc(PyCFunctionObject *m) +meth_dealloc(PyObject *self) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); // The Py_TRASHCAN mechanism requires that we be able to // call PyObject_GC_UnTrack twice on an object. PyObject_GC_UnTrack(m); @@ -175,8 +176,9 @@ meth_dealloc(PyCFunctionObject *m) } static PyObject * -meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) +meth_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); if (m->m_self == NULL || PyModule_Check(m->m_self)) return PyUnicode_FromString(m->m_ml->ml_name); @@ -185,32 +187,35 @@ meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) } static PyMethodDef meth_methods[] = { - {"__reduce__", (PyCFunction)meth_reduce, METH_NOARGS, NULL}, + {"__reduce__", meth_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; static PyObject * -meth_get__text_signature__(PyCFunctionObject *m, void *closure) +meth_get__text_signature__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc, m->m_ml->ml_flags); } static PyObject * -meth_get__doc__(PyCFunctionObject *m, void *closure) +meth_get__doc__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return _PyType_GetDocFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); } static PyObject * -meth_get__name__(PyCFunctionObject *m, void *closure) +meth_get__name__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return PyUnicode_FromString(m->m_ml->ml_name); } static PyObject * -meth_get__qualname__(PyCFunctionObject *m, void *closure) +meth_get__qualname__(PyObject *self, void *closure) { /* If __self__ is a module or NULL, return m.__name__ (e.g. len.__qualname__ == 'len') @@ -220,14 +225,15 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) Otherwise return type(m.__self__).__qualname__ + '.' + m.__name__ (e.g. [].append.__qualname__ == 'list.append') */ - PyObject *type, *type_qualname, *res; - if (m->m_self == NULL || PyModule_Check(m->m_self)) + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { return PyUnicode_FromString(m->m_ml->ml_name); + } - type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); + PyObject *type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); - type_qualname = PyObject_GetAttr(type, &_Py_ID(__qualname__)); + PyObject *type_qualname = PyObject_GetAttr(type, &_Py_ID(__qualname__)); if (type_qualname == NULL) return NULL; @@ -238,14 +244,15 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) return NULL; } - res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); + PyObject *res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); Py_DECREF(type_qualname); return res; } static int -meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) +meth_traverse(PyObject *self, visitproc visit, void *arg) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); Py_VISIT(PyCFunction_GET_CLASS(m)); Py_VISIT(m->m_self); Py_VISIT(m->m_module); @@ -253,22 +260,22 @@ meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) } static PyObject * -meth_get__self__(PyCFunctionObject *m, void *closure) +meth_get__self__(PyObject *meth, void *closure) { - PyObject *self; - - self = PyCFunction_GET_SELF(m); - if (self == NULL) + PyCFunctionObject *m = _PyCFunctionObject_CAST(meth); + PyObject *self = PyCFunction_GET_SELF(m); + if (self == NULL) { self = Py_None; + } return Py_NewRef(self); } -static PyGetSetDef meth_getsets [] = { - {"__doc__", (getter)meth_get__doc__, NULL, NULL}, - {"__name__", (getter)meth_get__name__, NULL, NULL}, - {"__qualname__", (getter)meth_get__qualname__, NULL, NULL}, - {"__self__", (getter)meth_get__self__, NULL, NULL}, - {"__text_signature__", (getter)meth_get__text_signature__, NULL, NULL}, +static PyGetSetDef meth_getsets[] = { + {"__doc__", meth_get__doc__, NULL, NULL}, + {"__name__", meth_get__name__, NULL, NULL}, + {"__qualname__", meth_get__qualname__, NULL, NULL}, + {"__self__", meth_get__self__, NULL, NULL}, + {"__text_signature__", meth_get__text_signature__, NULL, NULL}, {0} }; @@ -280,15 +287,18 @@ static PyMemberDef meth_members[] = { }; static PyObject * -meth_repr(PyCFunctionObject *m) +meth_repr(PyObject *self) { - if (m->m_self == NULL || PyModule_Check(m->m_self)) + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { return PyUnicode_FromFormat("", - m->m_ml->ml_name); + m->m_ml->ml_name); + } + return PyUnicode_FromFormat("", - m->m_ml->ml_name, - Py_TYPE(m->m_self)->tp_name, - m->m_self); + m->m_ml->ml_name, + Py_TYPE(m->m_self)->tp_name, + m->m_self); } static PyObject * @@ -317,14 +327,15 @@ meth_richcompare(PyObject *self, PyObject *other, int op) } static Py_hash_t -meth_hash(PyCFunctionObject *a) +meth_hash(PyObject *self) { - Py_hash_t x, y; - x = PyObject_GenericHash(a->m_self); - y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); + PyCFunctionObject *a = _PyCFunctionObject_CAST(self); + Py_hash_t x = PyObject_GenericHash(a->m_self); + Py_hash_t y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); x ^= y; - if (x == -1) + if (x == -1) { x = -2; + } return x; } @@ -334,16 +345,16 @@ PyTypeObject PyCFunction_Type = { "builtin_function_or_method", sizeof(PyCFunctionObject), 0, - (destructor)meth_dealloc, /* tp_dealloc */ + meth_dealloc, /* tp_dealloc */ offsetof(PyCFunctionObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)meth_repr, /* tp_repr */ + meth_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)meth_hash, /* tp_hash */ + meth_hash, /* tp_hash */ cfunction_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -352,7 +363,7 @@ PyTypeObject PyCFunction_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)meth_traverse, /* tp_traverse */ + meth_traverse, /* tp_traverse */ 0, /* tp_clear */ meth_richcompare, /* tp_richcompare */ offsetof(PyCFunctionObject, m_weakreflist), /* tp_weaklistoffset */ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index efc74dafb5fc73..f63ae4e048bcd9 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -15,6 +15,10 @@ #include "osdefs.h" // MAXPATHLEN +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) + + static PyMemberDef module_members[] = { {"__dict__", _Py_T_OBJECT, offsetof(PyModuleObject, md_dict), Py_READONLY}, {0} @@ -225,7 +229,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } name = _PyImport_ResolveNameWithPackageContext(name); - if ((m = (PyModuleObject*)PyModule_New(name)) == NULL) + + m = (PyModuleObject*)PyModule_New(name); + if (m == NULL) return NULL; if (module->m_size > 0) { @@ -758,22 +764,26 @@ module___init___impl(PyModuleObject *self, PyObject *name, PyObject *doc) } static void -module_dealloc(PyModuleObject *m) +module_dealloc(PyObject *self) { - int verbose = _Py_GetConfig()->verbose; + PyModuleObject *m = _PyModule_CAST(self); PyObject_GC_UnTrack(m); + + int verbose = _Py_GetConfig()->verbose; if (verbose && m->md_name) { PySys_FormatStderr("# destroy %U\n", m->md_name); } if (m->md_weaklist != NULL) PyObject_ClearWeakRefs((PyObject *) m); + /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_free && (m->md_def->m_size <= 0 || m->md_state != NULL)) { m->md_def->m_free(m); } + Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); if (m->md_state != NULL) @@ -782,8 +792,9 @@ module_dealloc(PyModuleObject *m) } static PyObject * -module_repr(PyModuleObject *m) +module_repr(PyObject *self) { + PyModuleObject *m = _PyModule_CAST(self); PyInterpreterState *interp = _PyInterpreterState_GET(); return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m); } @@ -1062,14 +1073,17 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyObject* -_Py_module_getattro(PyModuleObject *m, PyObject *name) +_Py_module_getattro(PyObject *self, PyObject *name) { + PyModuleObject *m = _PyModule_CAST(self); return _Py_module_getattro_impl(m, name, 0); } static int -module_traverse(PyModuleObject *m, visitproc visit, void *arg) +module_traverse(PyObject *self, visitproc visit, void *arg) { + PyModuleObject *m = _PyModule_CAST(self); + /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_traverse && (m->md_def->m_size <= 0 || m->md_state != NULL)) @@ -1078,13 +1092,16 @@ module_traverse(PyModuleObject *m, visitproc visit, void *arg) if (res) return res; } + Py_VISIT(m->md_dict); return 0; } static int -module_clear(PyModuleObject *m) +module_clear(PyObject *self) { + PyModuleObject *m = _PyModule_CAST(self); + /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_clear && (m->md_def->m_size <= 0 || m->md_state != NULL)) @@ -1149,8 +1166,10 @@ module_get_dict(PyModuleObject *m) } static PyObject * -module_get_annotate(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) { + PyModuleObject *m = _PyModule_CAST(self); + PyObject *dict = module_get_dict(m); if (dict == NULL) { return NULL; @@ -1168,12 +1187,14 @@ module_get_annotate(PyModuleObject *m, void *Py_UNUSED(ignored)) } static int -module_set_annotate(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +module_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyModuleObject *m = _PyModule_CAST(self); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); return -1; } + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; @@ -1200,8 +1221,10 @@ module_set_annotate(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored) } static PyObject * -module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) { + PyModuleObject *m = _PyModule_CAST(self); + PyObject *dict = module_get_dict(m); if (dict == NULL) { return NULL; @@ -1249,14 +1272,16 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) } static int -module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +module_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { - int ret = -1; + PyModuleObject *m = _PyModule_CAST(self); + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; } + int ret = -1; if (value != NULL) { /* set */ ret = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); @@ -1282,8 +1307,8 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor static PyGetSetDef module_getsets[] = { - {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, - {"__annotate__", (getter)module_get_annotate, (setter)module_set_annotate}, + {"__annotations__", module_get_annotations, module_set_annotations}, + {"__annotate__", module_get_annotate, module_set_annotate}, {NULL} }; @@ -1292,26 +1317,26 @@ PyTypeObject PyModule_Type = { "module", /* tp_name */ sizeof(PyModuleObject), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)module_dealloc, /* tp_dealloc */ + module_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)module_repr, /* tp_repr */ + module_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)_Py_module_getattro, /* tp_getattro */ + _Py_module_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ module___init____doc__, /* tp_doc */ - (traverseproc)module_traverse, /* tp_traverse */ - (inquiry)module_clear, /* tp_clear */ + module_traverse, /* tp_traverse */ + module_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyModuleObject, md_weaklist), /* tp_weaklistoffset */ 0, /* tp_iter */ diff --git a/Objects/object.c b/Objects/object.c index 8a819dd336e421..4a4c5bf7d7f08a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -862,6 +862,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // stacks during GC, so emptying the free-list is counterproductive. clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); } + clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); } /* @@ -2371,6 +2372,14 @@ _PyTypes_InitTypes(PyInterpreterState *interp) } } + // Cache __reduce__ from PyBaseObject_Type object + PyObject *baseobj_dict = _PyType_GetDict(&PyBaseObject_Type); + PyObject *baseobj_reduce = PyDict_GetItemWithError(baseobj_dict, &_Py_ID(__reduce__)); + if (baseobj_reduce == NULL && PyErr_Occurred()) { + return _PyStatus_ERR("Can't get __reduce__ from base object"); + } + _Py_INTERP_CACHED_OBJECT(interp, objreduce) = baseobj_reduce; + // Must be after static types are initialized if (_Py_initialize_generic(interp) < 0) { return _PyStatus_ERR("Can't initialize generic types"); @@ -2452,7 +2461,7 @@ _Py_SetImmortalUntracked(PyObject *op) op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; op->ob_ref_shared = 0; #else - op->ob_refcnt = _Py_IMMORTAL_REFCNT; + op->ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; #endif } @@ -2491,6 +2500,42 @@ _Py_ResurrectReference(PyObject *op) #ifdef Py_TRACE_REFS +/* Make sure the ref is associated with the right interpreter. + * This only needs special attention for heap-allocated objects + * that have been immortalized, and only when the object might + * outlive the interpreter where it was created. That means the + * object was necessarily created using a global allocator + * (i.e. from the main interpreter). Thus in that specific case + * we move the object over to the main interpreter's refchain. + * + * This was added for the sake of the immortal interned strings, + * where legacy subinterpreters share the main interpreter's + * interned dict (and allocator), and therefore the strings can + * outlive the subinterpreter. + * + * It may make sense to fold this into _Py_SetImmortalUntracked(), + * but that requires further investigation. In the meantime, it is + * up to the caller to know if this is needed. There should be + * very few cases. + */ +void +_Py_NormalizeImmortalReference(PyObject *op) +{ + assert(_Py_IsImmortal(op)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!_PyRefchain_IsTraced(interp, op)) { + return; + } + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + if (interp != main_interp + && interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) + { + assert(!_PyRefchain_IsTraced(main_interp, op)); + _PyRefchain_Remove(interp, op); + _PyRefchain_Trace(main_interp, op); + } +} + void _Py_ForgetReference(PyObject *op) { diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1318ce0319d438..2942ab624edf72 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -143,14 +143,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw) static PyObject * -range_vectorcall(PyTypeObject *type, PyObject *const *args, +range_vectorcall(PyObject *rangetype, PyObject *const *args, size_t nargsf, PyObject *kwnames) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (!_PyArg_NoKwnames("range", kwnames)) { return NULL; } - return range_from_array(type, args, nargs); + return range_from_array((PyTypeObject *)rangetype, args, nargs); } PyDoc_STRVAR(range_doc, @@ -803,7 +803,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_init */ 0, /* tp_alloc */ range_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)range_vectorcall + .tp_vectorcall = range_vectorcall }; /*********************** range Iterator **************************/ diff --git a/Objects/setobject.c b/Objects/setobject.c index c5f96d25585fa4..9f40e085f06fa6 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -40,6 +40,8 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition + +#include "stringlib/eq.h" // unicode_eq() #include // offsetof() #include "clinic/setobject.c.h" @@ -96,7 +98,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) return entry; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && _PyUnicode_EQ(startkey, key)) + && unicode_eq(startkey, key)) return entry; table = so->table; Py_INCREF(startkey); @@ -157,7 +159,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && _PyUnicode_EQ(startkey, key)) + && unicode_eq(startkey, key)) goto found_active; table = so->table; Py_INCREF(startkey); @@ -404,8 +406,9 @@ set_empty_to_minsize(PySetObject *so) } static int -set_clear_internal(PySetObject *so) +set_clear_internal(PyObject *self) { + PySetObject *so = _PySet_CAST(self); setentry *entry; setentry *table = so->table; Py_ssize_t fill = so->fill; @@ -490,8 +493,9 @@ set_next(PySetObject *so, Py_ssize_t *pos_ptr, setentry **entry_ptr) } static void -set_dealloc(PySetObject *so) +set_dealloc(PyObject *self) { + PySetObject *so = _PySet_CAST(self); setentry *entry; Py_ssize_t used = so->used; @@ -559,8 +563,9 @@ set_repr_lock_held(PySetObject *so) } static PyObject * -set_repr(PySetObject *so) +set_repr(PyObject *self) { + PySetObject *so = _PySet_CAST(self); PyObject *result; Py_BEGIN_CRITICAL_SECTION(so); result = set_repr_lock_held(so); @@ -569,8 +574,9 @@ set_repr(PySetObject *so) } static Py_ssize_t -set_len(PySetObject *so) +set_len(PyObject *self) { + PySetObject *so = _PySet_CAST(self); return FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); } @@ -584,11 +590,10 @@ set_merge_lock_held(PySetObject *so, PyObject *otherset) setentry *other_entry; assert (PyAnySet_Check(so)); - assert (PyAnySet_Check(otherset)); _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(otherset); - other = (PySetObject*)otherset; + other = _PySet_CAST(otherset); if (other == so || other->used == 0) /* a.update(a) or a.update(set()); nothing to do */ return 0; @@ -684,8 +689,9 @@ set_pop_impl(PySetObject *so) } static int -set_traverse(PySetObject *so, visitproc visit, void *arg) +set_traverse(PyObject *self, visitproc visit, void *arg) { + PySetObject *so = _PySet_CAST(self); Py_ssize_t pos = 0; setentry *entry; @@ -718,8 +724,7 @@ _shuffle_bits(Py_uhash_t h) static Py_hash_t frozenset_hash_impl(PyObject *self) { - assert(PyAnySet_Check(self)); - PySetObject *so = (PySetObject *)self; + PySetObject *so = _PySet_CAST(self); Py_uhash_t hash = 0; setentry *entry; @@ -761,7 +766,7 @@ frozenset_hash_impl(PyObject *self) static Py_hash_t frozenset_hash(PyObject *self) { - PySetObject *so = (PySetObject *)self; + PySetObject *so = _PySet_CAST(self); Py_uhash_t hash; if (so->hash != -1) { @@ -784,8 +789,9 @@ typedef struct { } setiterobject; static void -setiter_dealloc(setiterobject *si) +setiter_dealloc(PyObject *self) { + setiterobject *si = (setiterobject*)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(si); Py_XDECREF(si->si_set); @@ -793,8 +799,9 @@ setiter_dealloc(setiterobject *si) } static int -setiter_traverse(setiterobject *si, visitproc visit, void *arg) +setiter_traverse(PyObject *self, visitproc visit, void *arg) { + setiterobject *si = (setiterobject*)self; Py_VISIT(si->si_set); return 0; } @@ -810,8 +817,6 @@ setiter_len(setiterobject *si, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject *setiter_iternext(setiterobject *si); - static PyObject * setiter_reduce(setiterobject *si, PyObject *Py_UNUSED(ignored)) { @@ -836,8 +841,9 @@ static PyMethodDef setiter_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyObject *setiter_iternext(setiterobject *si) +static PyObject *setiter_iternext(PyObject *self) { + setiterobject *si = (setiterobject*)self; PyObject *key = NULL; Py_ssize_t i, mask; setentry *entry; @@ -884,7 +890,7 @@ PyTypeObject PySetIter_Type = { sizeof(setiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)setiter_dealloc, /* tp_dealloc */ + setiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -901,18 +907,18 @@ PyTypeObject PySetIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)setiter_traverse, /* tp_traverse */ + setiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)setiter_iternext, /* tp_iternext */ + setiter_iternext, /* tp_iternext */ setiter_methods, /* tp_methods */ 0, }; static PyObject * -set_iter(PySetObject *so) +set_iter(PyObject *so) { Py_ssize_t size = set_len(so); setiterobject *si = PyObject_GC_New(setiterobject, &PySetIter_Type); @@ -1270,7 +1276,7 @@ static PyObject * set_clear_impl(PySetObject *so) /*[clinic end generated code: output=4e71d5a83904161a input=c6f831b366111950]*/ { - set_clear_internal(so); + set_clear_internal((PyObject*)so); Py_RETURN_NONE; } @@ -1307,12 +1313,13 @@ set_union_impl(PySetObject *so, PyObject *args) } static PyObject * -set_or(PySetObject *so, PyObject *other) +set_or(PyObject *self, PyObject *other) { PySetObject *result; - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + if (!PyAnySet_Check(self) || !PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); result = (PySetObject *)set_copy(so, NULL); if (result == NULL) { @@ -1329,10 +1336,11 @@ set_or(PySetObject *so, PyObject *other) } static PyObject * -set_ior(PySetObject *so, PyObject *other) +set_ior(PyObject *self, PyObject *other) { if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); if (set_update_internal(so, other)) { return NULL; @@ -1495,10 +1503,11 @@ set_intersection_update_multi_impl(PySetObject *so, PyObject *args) } static PyObject * -set_and(PySetObject *so, PyObject *other) +set_and(PyObject *self, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + if (!PyAnySet_Check(self) || !PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); PyObject *rv; Py_BEGIN_CRITICAL_SECTION2(so, other); @@ -1509,12 +1518,13 @@ set_and(PySetObject *so, PyObject *other) } static PyObject * -set_iand(PySetObject *so, PyObject *other) +set_iand(PyObject *self, PyObject *other) { PyObject *result; if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); Py_BEGIN_CRITICAL_SECTION2(so, other); result = set_intersection_update(so, other); @@ -1603,7 +1613,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); if ((PyObject *)so == other) - return set_clear_internal(so); + return set_clear_internal((PyObject*)so); if (PyAnySet_Check(other)) { setentry *entry; @@ -1815,10 +1825,11 @@ set_difference_multi_impl(PySetObject *so, PyObject *args) } static PyObject * -set_sub(PySetObject *so, PyObject *other) +set_sub(PyObject *self, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + if (!PyAnySet_Check(self) || !PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); PyObject *rv; Py_BEGIN_CRITICAL_SECTION2(so, other); @@ -1828,10 +1839,11 @@ set_sub(PySetObject *so, PyObject *other) } static PyObject * -set_isub(PySetObject *so, PyObject *other) +set_isub(PyObject *self, PyObject *other) { if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); int rv; Py_BEGIN_CRITICAL_SECTION2(so, other); @@ -1973,20 +1985,23 @@ set_symmetric_difference_impl(PySetObject *so, PyObject *other) } static PyObject * -set_xor(PySetObject *so, PyObject *other) +set_xor(PyObject *self, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + if (!PyAnySet_Check(self) || !PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); return set_symmetric_difference(so, other); } static PyObject * -set_ixor(PySetObject *so, PyObject *other) +set_ixor(PyObject *self, PyObject *other) { PyObject *result; if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); + result = set_symmetric_difference_update(so, other); if (result == NULL) return NULL; @@ -2081,8 +2096,9 @@ set_issuperset_impl(PySetObject *so, PyObject *other) } static PyObject * -set_richcompare(PySetObject *v, PyObject *w, int op) +set_richcompare(PyObject *self, PyObject *w, int op) { + PySetObject *v = _PySet_CAST(self); PyObject *r1; int r2; @@ -2099,7 +2115,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_FALSE; return set_issubset(v, w); case Py_NE: - r1 = set_richcompare(v, w, Py_EQ); + r1 = set_richcompare((PyObject*)v, w, Py_EQ); if (r1 == NULL) return NULL; r2 = PyObject_IsTrue(r1); @@ -2173,6 +2189,13 @@ _PySet_Contains(PySetObject *so, PyObject *key) return rv; } +static int +set_contains(PyObject *self, PyObject *key) +{ + PySetObject *so = _PySet_CAST(self); + return _PySet_Contains(so, key); +} + /*[clinic input] @critical_section @coexist @@ -2321,8 +2344,9 @@ set___sizeof___impl(PySetObject *so) } static int -set_init(PySetObject *self, PyObject *args, PyObject *kwds) +set_init(PyObject *so, PyObject *args, PyObject *kwds) { + PySetObject *self = _PySet_CAST(so); PyObject *iterable = NULL; if (!_PyArg_NoKeywords("set", kwds)) @@ -2339,7 +2363,7 @@ set_init(PySetObject *self, PyObject *args, PyObject *kwds) } Py_BEGIN_CRITICAL_SECTION(self); if (self->fill) - set_clear_internal(self); + set_clear_internal((PyObject*)self); self->hash = -1; Py_END_CRITICAL_SECTION(); @@ -2371,14 +2395,14 @@ set_vectorcall(PyObject *type, PyObject * const*args, } static PySequenceMethods set_as_sequence = { - (lenfunc)set_len, /* sq_length */ + set_len, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ 0, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)_PySet_Contains, /* sq_contains */ + set_contains, /* sq_contains */ }; /* set object ********************************************************/ @@ -2410,7 +2434,7 @@ static PyMethodDef set_methods[] = { static PyNumberMethods set_as_number = { 0, /*nb_add*/ - (binaryfunc)set_sub, /*nb_subtract*/ + set_sub, /*nb_subtract*/ 0, /*nb_multiply*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ @@ -2422,22 +2446,22 @@ static PyNumberMethods set_as_number = { 0, /*nb_invert*/ 0, /*nb_lshift*/ 0, /*nb_rshift*/ - (binaryfunc)set_and, /*nb_and*/ - (binaryfunc)set_xor, /*nb_xor*/ - (binaryfunc)set_or, /*nb_or*/ + set_and, /*nb_and*/ + set_xor, /*nb_xor*/ + set_or, /*nb_or*/ 0, /*nb_int*/ 0, /*nb_reserved*/ 0, /*nb_float*/ 0, /*nb_inplace_add*/ - (binaryfunc)set_isub, /*nb_inplace_subtract*/ + set_isub, /*nb_inplace_subtract*/ 0, /*nb_inplace_multiply*/ 0, /*nb_inplace_remainder*/ 0, /*nb_inplace_power*/ 0, /*nb_inplace_lshift*/ 0, /*nb_inplace_rshift*/ - (binaryfunc)set_iand, /*nb_inplace_and*/ - (binaryfunc)set_ixor, /*nb_inplace_xor*/ - (binaryfunc)set_ior, /*nb_inplace_or*/ + set_iand, /*nb_inplace_and*/ + set_ixor, /*nb_inplace_xor*/ + set_ior, /*nb_inplace_or*/ }; PyDoc_STRVAR(set_doc, @@ -2452,12 +2476,12 @@ PyTypeObject PySet_Type = { sizeof(PySetObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)set_dealloc, /* tp_dealloc */ + set_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)set_repr, /* tp_repr */ + set_repr, /* tp_repr */ &set_as_number, /* tp_as_number */ &set_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -2469,13 +2493,13 @@ PyTypeObject PySet_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | - _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ + _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ set_doc, /* tp_doc */ - (traverseproc)set_traverse, /* tp_traverse */ - (inquiry)set_clear_internal, /* tp_clear */ - (richcmpfunc)set_richcompare, /* tp_richcompare */ + set_traverse, /* tp_traverse */ + set_clear_internal, /* tp_clear */ + set_richcompare, /* tp_richcompare */ offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)set_iter, /* tp_iter */ + set_iter, /* tp_iter */ 0, /* tp_iternext */ set_methods, /* tp_methods */ 0, /* tp_members */ @@ -2485,7 +2509,7 @@ PyTypeObject PySet_Type = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)set_init, /* tp_init */ + set_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ set_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ @@ -2513,7 +2537,7 @@ static PyMethodDef frozenset_methods[] = { static PyNumberMethods frozenset_as_number = { 0, /*nb_add*/ - (binaryfunc)set_sub, /*nb_subtract*/ + set_sub, /*nb_subtract*/ 0, /*nb_multiply*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ @@ -2525,9 +2549,9 @@ static PyNumberMethods frozenset_as_number = { 0, /*nb_invert*/ 0, /*nb_lshift*/ 0, /*nb_rshift*/ - (binaryfunc)set_and, /*nb_and*/ - (binaryfunc)set_xor, /*nb_xor*/ - (binaryfunc)set_or, /*nb_or*/ + set_and, /*nb_and*/ + set_xor, /*nb_xor*/ + set_or, /*nb_or*/ }; PyDoc_STRVAR(frozenset_doc, @@ -2542,12 +2566,12 @@ PyTypeObject PyFrozenSet_Type = { sizeof(PySetObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)set_dealloc, /* tp_dealloc */ + set_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)set_repr, /* tp_repr */ + set_repr, /* tp_repr */ &frozenset_as_number, /* tp_as_number */ &set_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -2559,13 +2583,13 @@ PyTypeObject PyFrozenSet_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | - _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ + _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ frozenset_doc, /* tp_doc */ - (traverseproc)set_traverse, /* tp_traverse */ - (inquiry)set_clear_internal, /* tp_clear */ - (richcmpfunc)set_richcompare, /* tp_richcompare */ + set_traverse, /* tp_traverse */ + set_clear_internal, /* tp_clear */ + set_richcompare, /* tp_richcompare */ offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)set_iter, /* tp_iter */ + set_iter, /* tp_iter */ 0, /* tp_iternext */ frozenset_methods, /* tp_methods */ 0, /* tp_members */ @@ -2604,7 +2628,7 @@ PySet_Size(PyObject *anyset) PyErr_BadInternalCall(); return -1; } - return set_len((PySetObject *)anyset); + return set_len(anyset); } int @@ -2621,7 +2645,7 @@ PySet_Clear(PyObject *set) void _PySet_ClearInternal(PySetObject *so) { - (void)set_clear_internal(so); + (void)set_clear_internal((PyObject*)so); } int diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 1b6d35998c2b69..4fef0af93fe095 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -343,7 +343,7 @@ Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PySliceObject *r) { - _PyObject_GC_UNTRACK(r); + PyObject_GC_UnTrack(r); Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); diff --git a/Objects/stringlib/eq.h b/Objects/stringlib/eq.h index 2eac4baf5ca9ce..821b692f26b830 100644 --- a/Objects/stringlib/eq.h +++ b/Objects/stringlib/eq.h @@ -4,14 +4,19 @@ * unicode_eq() is called when the hash of two unicode objects is equal. */ Py_LOCAL_INLINE(int) -unicode_eq(PyObject *a, PyObject *b) +unicode_eq(PyObject *str1, PyObject *str2) { - if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b)) + Py_ssize_t len = PyUnicode_GET_LENGTH(str1); + if (PyUnicode_GET_LENGTH(str2) != len) { return 0; - if (PyUnicode_GET_LENGTH(a) == 0) - return 1; - if (PyUnicode_KIND(a) != PyUnicode_KIND(b)) + } + + int kind = PyUnicode_KIND(str1); + if (PyUnicode_KIND(str2) != kind) { return 0; - return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b), - PyUnicode_GET_LENGTH(a) * PyUnicode_KIND(a)) == 0; + } + + const void *data1 = PyUnicode_DATA(str1); + const void *data2 = PyUnicode_DATA(str2); + return (memcmp(data1, data2, len * kind) == 0); } diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index 91c71ab5736c25..44b269ba8ceb55 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -73,7 +73,7 @@ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { if (str->str == NULL) { - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } return SubString_new_object(str); } @@ -531,7 +531,7 @@ render_field(PyObject *fieldobj, SubString *format_spec, _PyUnicodeWriter *write format_spec->start, format_spec->end); else - format_spec_object = PyUnicode_New(0, 0); + format_spec_object = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (format_spec_object == NULL) goto done; diff --git a/Objects/structseq.c b/Objects/structseq.c index ee3dbf9d4c047a..56a7851b98788d 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -266,83 +266,66 @@ static PyObject * structseq_repr(PyStructSequence *obj) { PyTypeObject *typ = Py_TYPE(obj); - _PyUnicodeWriter writer; - /* Write "typename(" */ - PyObject *type_name = PyUnicode_DecodeUTF8(typ->tp_name, - strlen(typ->tp_name), - NULL); - if (type_name == NULL) { + // count 5 characters per item: "x=1, " + Py_ssize_t type_name_len = strlen(typ->tp_name); + Py_ssize_t prealloc = (type_name_len + 1 + + VISIBLE_SIZE(obj) * 5 + 1); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { return NULL; } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - /* count 5 characters per item: "x=1, " */ - writer.min_length = (PyUnicode_GET_LENGTH(type_name) + 1 - + VISIBLE_SIZE(obj) * 5 + 1); - - if (_PyUnicodeWriter_WriteStr(&writer, type_name) < 0) { - Py_DECREF(type_name); + // Write "typename(" + if (PyUnicodeWriter_WriteUTF8(writer, typ->tp_name, type_name_len) < 0) { goto error; } - Py_DECREF(type_name); - - if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { goto error; } for (Py_ssize_t i=0; i < VISIBLE_SIZE(obj); i++) { if (i > 0) { - /* Write ", " */ - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + // Write ", " + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; } } - /* Write "name=repr" */ + // Write name const char *name_utf8 = typ->tp_members[i].name; if (name_utf8 == NULL) { - PyErr_Format(PyExc_SystemError, "In structseq_repr(), member %zd name is NULL" + PyErr_Format(PyExc_SystemError, + "In structseq_repr(), member %zd name is NULL" " for type %.500s", i, typ->tp_name); goto error; } - - PyObject *name = PyUnicode_DecodeUTF8(name_utf8, strlen(name_utf8), NULL); - if (name == NULL) { - goto error; - } - if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { - Py_DECREF(name); + if (PyUnicodeWriter_WriteUTF8(writer, name_utf8, -1) < 0) { goto error; } - Py_DECREF(name); - if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { + // Write "=" + repr(value) + if (PyUnicodeWriter_WriteChar(writer, '=') < 0) { goto error; } - PyObject *value = PyStructSequence_GetItem((PyObject*)obj, i); assert(value != NULL); - PyObject *repr = PyObject_Repr(value); - if (repr == NULL) { - goto error; - } - if (_PyUnicodeWriter_WriteStr(&writer, repr) < 0) { - Py_DECREF(repr); + if (PyUnicodeWriter_WriteRepr(writer, value) < 0) { goto error; } - Py_DECREF(repr); } - if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -725,7 +708,7 @@ _PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) assert(type->tp_name != NULL); assert(type->tp_base == &PyTuple_Type); assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); - assert(_Py_IsImmortalLoose(type)); + assert(_Py_IsImmortal(type)); // Cannot delete a type if it still has subclasses if (_PyType_HasSubclasses(type)) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f14f10ab9c0a46..f3132e0933ac30 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -181,8 +181,9 @@ PyTuple_Pack(Py_ssize_t n, ...) /* Methods */ static void -tupledealloc(PyTupleObject *op) +tuple_dealloc(PyObject *self) { + PyTupleObject *op = _PyTuple_CAST(self); if (Py_SIZE(op) == 0) { /* The empty tuple is statically allocated. */ if (op == &_Py_SINGLETON(tuple_empty)) { @@ -199,7 +200,7 @@ tupledealloc(PyTupleObject *op) } PyObject_GC_UnTrack(op); - Py_TRASHCAN_BEGIN(op, tupledealloc) + Py_TRASHCAN_BEGIN(op, tuple_dealloc) Py_ssize_t i = Py_SIZE(op); while (--i >= 0) { @@ -214,73 +215,71 @@ tupledealloc(PyTupleObject *op) } static PyObject * -tuplerepr(PyTupleObject *v) +tuple_repr(PyObject *self) { - Py_ssize_t i, n; - _PyUnicodeWriter writer; - - n = Py_SIZE(v); - if (n == 0) + PyTupleObject *v = _PyTuple_CAST(self); + Py_ssize_t n = PyTuple_GET_SIZE(v); + if (n == 0) { return PyUnicode_FromString("()"); + } /* While not mutable, it is still possible to end up with a cycle in a tuple through an object that stores itself within a tuple (and thus infinitely asks for the repr of itself). This should only be possible within a type. */ - i = Py_ReprEnter((PyObject *)v); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("(...)") : NULL; + int res = Py_ReprEnter((PyObject *)v); + if (res != 0) { + return res > 0 ? PyUnicode_FromString("(...)") : NULL; } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - if (Py_SIZE(v) > 1) { - /* "(" + "1" + ", 2" * (len - 1) + ")" */ - writer.min_length = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + Py_ssize_t prealloc; + if (n > 1) { + // "(" + "1" + ", 2" * (len - 1) + ")" + prealloc = 1 + 1 + (2 + 1) * (n - 1) + 1; } else { - /* "(1,)" */ - writer.min_length = 4; + // "(1,)" + prealloc = 4; + } + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; } - if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { goto error; + } /* Do repr() on each element. */ - for (i = 0; i < n; ++i) { - PyObject *s; - + for (Py_ssize_t i = 0; i < n; ++i) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; + } } - s = PyObject_Repr(v->ob_item[i]); - if (s == NULL) - goto error; - - if (_PyUnicodeWriter_WriteStr(&writer, s) < 0) { - Py_DECREF(s); + if (PyUnicodeWriter_WriteRepr(writer, v->ob_item[i]) < 0) { goto error; } - Py_DECREF(s); } - writer.overallocate = 0; - if (n > 1) { - if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) + if (n == 1) { + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { goto error; + } } - else { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ",)", 2) < 0) - goto error; + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + goto error; } Py_ReprLeave((PyObject *)v); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_ReprLeave((PyObject *)v); return NULL; } @@ -316,13 +315,14 @@ tuplerepr(PyTupleObject *v) /* Tests have shown that it's not worth to cache the hash value, see https://bugs.python.org/issue9685 */ static Py_hash_t -tuplehash(PyTupleObject *v) +tuple_hash(PyObject *op) { - Py_ssize_t i, len = Py_SIZE(v); + PyTupleObject *v = _PyTuple_CAST(op); + Py_ssize_t len = Py_SIZE(v); PyObject **item = v->ob_item; Py_uhash_t acc = _PyHASH_XXPRIME_5; - for (i = 0; i < len; i++) { + for (Py_ssize_t i = 0; i < len; i++) { Py_uhash_t lane = PyObject_Hash(item[i]); if (lane == (Py_uhash_t)-1) { return -1; @@ -342,25 +342,27 @@ tuplehash(PyTupleObject *v) } static Py_ssize_t -tuplelength(PyTupleObject *a) +tuple_length(PyObject *self) { + PyTupleObject *a = _PyTuple_CAST(self); return Py_SIZE(a); } static int -tuplecontains(PyTupleObject *a, PyObject *el) +tuple_contains(PyObject *self, PyObject *el) { - Py_ssize_t i; - int cmp; - - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) + PyTupleObject *a = _PyTuple_CAST(self); + int cmp = 0; + for (Py_ssize_t i = 0; cmp == 0 && i < Py_SIZE(a); ++i) { cmp = PyObject_RichCompareBool(PyTuple_GET_ITEM(a, i), el, Py_EQ); + } return cmp; } static PyObject * -tupleitem(PyTupleObject *a, Py_ssize_t i) +tuple_item(PyObject *op, Py_ssize_t i) { + PyTupleObject *a = _PyTuple_CAST(op); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "tuple index out of range"); return NULL; @@ -432,7 +434,7 @@ _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) } static PyObject * -tupleslice(PyTupleObject *a, Py_ssize_t ilow, +tuple_slice(PyTupleObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { if (ilow < 0) @@ -454,16 +456,13 @@ PyTuple_GetSlice(PyObject *op, Py_ssize_t i, Py_ssize_t j) PyErr_BadInternalCall(); return NULL; } - return tupleslice((PyTupleObject *)op, i, j); + return tuple_slice((PyTupleObject *)op, i, j); } static PyObject * -tupleconcat(PyTupleObject *a, PyObject *bb) +tuple_concat(PyObject *aa, PyObject *bb) { - Py_ssize_t size; - Py_ssize_t i; - PyObject **src, **dest; - PyTupleObject *np; + PyTupleObject *a = _PyTuple_CAST(aa); if (Py_SIZE(a) == 0 && PyTuple_CheckExact(bb)) { return Py_NewRef(bb); } @@ -479,34 +478,38 @@ tupleconcat(PyTupleObject *a, PyObject *bb) return Py_NewRef(a); } assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); - size = Py_SIZE(a) + Py_SIZE(b); + Py_ssize_t size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { return tuple_get_empty(); } - np = tuple_alloc(size); + PyTupleObject *np = tuple_alloc(size); if (np == NULL) { return NULL; } - src = a->ob_item; - dest = np->ob_item; - for (i = 0; i < Py_SIZE(a); i++) { + + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (Py_ssize_t i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; dest[i] = Py_NewRef(v); } + src = b->ob_item; dest = np->ob_item + Py_SIZE(a); - for (i = 0; i < Py_SIZE(b); i++) { + for (Py_ssize_t i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; dest[i] = Py_NewRef(v); } + _PyObject_GC_TRACK(np); return (PyObject *)np; } static PyObject * -tuplerepeat(PyTupleObject *a, Py_ssize_t n) +tuple_repeat(PyObject *self, Py_ssize_t n) { + PyTupleObject *a = _PyTuple_CAST(self); const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n == 1) { if (PyTuple_CheckExact(a)) { @@ -621,17 +624,17 @@ tuple_count(PyTupleObject *self, PyObject *value) } static int -tupletraverse(PyTupleObject *o, visitproc visit, void *arg) +tuple_traverse(PyObject *self, visitproc visit, void *arg) { - Py_ssize_t i; - - for (i = Py_SIZE(o); --i >= 0; ) + PyTupleObject *o = _PyTuple_CAST(self); + for (Py_ssize_t i = Py_SIZE(o); --i >= 0; ) { Py_VISIT(o->ob_item[i]); + } return 0; } static PyObject * -tuplerichcompare(PyObject *v, PyObject *w, int op) +tuple_richcompare(PyObject *v, PyObject *w, int op) { PyTupleObject *vt, *wt; Py_ssize_t i; @@ -770,26 +773,27 @@ tuple_subtype_new(PyTypeObject *type, PyObject *iterable) } static PySequenceMethods tuple_as_sequence = { - (lenfunc)tuplelength, /* sq_length */ - (binaryfunc)tupleconcat, /* sq_concat */ - (ssizeargfunc)tuplerepeat, /* sq_repeat */ - (ssizeargfunc)tupleitem, /* sq_item */ + tuple_length, /* sq_length */ + tuple_concat, /* sq_concat */ + tuple_repeat, /* sq_repeat */ + tuple_item, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)tuplecontains, /* sq_contains */ + tuple_contains, /* sq_contains */ }; static PyObject* -tuplesubscript(PyTupleObject* self, PyObject* item) +tuple_subscript(PyObject *op, PyObject* item) { + PyTupleObject *self = _PyTuple_CAST(op); if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) i += PyTuple_GET_SIZE(self); - return tupleitem(self, i); + return tuple_item(op, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; @@ -843,7 +847,7 @@ static PyObject * tuple___getnewargs___impl(PyTupleObject *self) /*[clinic end generated code: output=25e06e3ee56027e2 input=1aeb4b286a21639a]*/ { - return Py_BuildValue("(N)", tupleslice(self, 0, Py_SIZE(self))); + return Py_BuildValue("(N)", tuple_slice(self, 0, Py_SIZE(self))); } static PyMethodDef tuple_methods[] = { @@ -855,8 +859,8 @@ static PyMethodDef tuple_methods[] = { }; static PyMappingMethods tuple_as_mapping = { - (lenfunc)tuplelength, - (binaryfunc)tuplesubscript, + tuple_length, + tuple_subscript, 0 }; @@ -867,16 +871,16 @@ PyTypeObject PyTuple_Type = { "tuple", sizeof(PyTupleObject) - sizeof(PyObject *), sizeof(PyObject *), - (destructor)tupledealloc, /* tp_dealloc */ + tuple_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)tuplerepr, /* tp_repr */ + tuple_repr, /* tp_repr */ 0, /* tp_as_number */ &tuple_as_sequence, /* tp_as_sequence */ &tuple_as_mapping, /* tp_as_mapping */ - (hashfunc)tuplehash, /* tp_hash */ + tuple_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -886,9 +890,9 @@ PyTypeObject PyTuple_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TUPLE_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, /* tp_flags */ tuple_new__doc__, /* tp_doc */ - (traverseproc)tupletraverse, /* tp_traverse */ + tuple_traverse, /* tp_traverse */ 0, /* tp_clear */ - tuplerichcompare, /* tp_richcompare */ + tuple_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ tuple_iter, /* tp_iter */ 0, /* tp_iternext */ @@ -999,8 +1003,9 @@ tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) } static PyObject * -tupleiter_next(_PyTupleIterObject *it) +tupleiter_next(PyObject *obj) { + _PyTupleIterObject *it = (_PyTupleIterObject *)obj; PyTupleObject *seq; PyObject *item; @@ -1101,7 +1106,7 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tupleiter_next, /* tp_iternext */ + tupleiter_next, /* tp_iternext */ tupleiter_methods, /* tp_methods */ 0, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 68e481f8e5163b..6ca4406ec0ea2d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -476,7 +476,7 @@ set_tp_bases(PyTypeObject *self, PyObject *bases, int initial) assert(PyTuple_GET_SIZE(bases) == 1); assert(PyTuple_GET_ITEM(bases, 0) == (PyObject *)self->tp_base); assert(self->tp_base->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - assert(_Py_IsImmortalLoose(self->tp_base)); + assert(_Py_IsImmortal(self->tp_base)); } _Py_SetImmortal(bases); } @@ -493,7 +493,7 @@ clear_tp_bases(PyTypeObject *self, int final) Py_CLEAR(self->tp_bases); } else { - assert(_Py_IsImmortalLoose(self->tp_bases)); + assert(_Py_IsImmortal(self->tp_bases)); _Py_ClearImmortal(self->tp_bases); } } @@ -558,7 +558,7 @@ clear_tp_mro(PyTypeObject *self, int final) Py_CLEAR(self->tp_mro); } else { - assert(_Py_IsImmortalLoose(self->tp_mro)); + assert(_Py_IsImmortal(self->tp_mro)); _Py_ClearImmortal(self->tp_mro); } } @@ -1435,6 +1435,9 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } @@ -3929,7 +3932,7 @@ type_new_alloc(type_new_ctx *ctx) et->ht_token = NULL; #ifdef Py_GIL_DISABLED - _PyType_AssignId(et); + et->unique_id = _PyObject_AssignUniqueId((PyObject *)et); #endif return type; @@ -5023,7 +5026,7 @@ PyType_FromMetaclass( #ifdef Py_GIL_DISABLED // Assign a type id to enable thread-local refcounting - _PyType_AssignId(res); + res->unique_id = _PyObject_AssignUniqueId((PyObject *)res); #endif /* Ready the type (which includes inheritance). @@ -5207,8 +5210,8 @@ PyType_GetModuleState(PyTypeObject *type) /* Get the module of the first superclass where the module has the * given PyModuleDef. */ -static inline PyObject * -get_module_by_def(PyTypeObject *type, PyModuleDef *def) +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); @@ -5241,7 +5244,7 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); - if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + if (!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped continue; } @@ -5254,46 +5257,22 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) } } END_TYPE_LOCK(); - return res; -} -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *module = get_module_by_def(type, def); - if (module == NULL) { + if (res == NULL) { PyErr_Format( PyExc_TypeError, "PyType_GetModuleByDef: No superclass of '%s' has the given module", type->tp_name); } - return module; -} - -PyObject * -_PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, - PyModuleDef *def) -{ - PyObject *module = get_module_by_def(left, def); - if (module == NULL) { - module = get_module_by_def(right, def); - if (module == NULL) { - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' nor '%s' has " - "the given module", left->tp_name, right->tp_name); - } - } - return module; + return res; } static PyTypeObject * -get_base_by_token_recursive(PyTypeObject *type, void *token) +get_base_by_token_recursive(PyObject *bases, void *token) { - assert(PyType_GetSlot(type, Py_tp_token) != token); - PyObject *bases = lookup_tp_bases(type); assert(bases != NULL); + PyTypeObject *res = NULL; Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); @@ -5301,112 +5280,76 @@ get_base_by_token_recursive(PyTypeObject *type, void *token) continue; } if (((PyHeapTypeObject*)base)->ht_token == token) { - return base; + res = base; + break; } - base = get_base_by_token_recursive(base, token); + base = get_base_by_token_recursive(lookup_tp_bases(base), token); if (base != NULL) { - return base; + res = base; + break; } } - return NULL; + return res; // Prefer to return recursively from one place } -static inline PyTypeObject * -get_base_by_token_from_mro(PyTypeObject *type, void *token) +int +PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) { - // Bypass lookup_tp_mro() as PyType_IsSubtype() does - PyObject *mro = type->tp_mro; - assert(mro != NULL); - assert(PyTuple_Check(mro)); - // mro_invoke() ensures that the type MRO cannot be empty. - assert(PyTuple_GET_SIZE(mro) >= 1); - // Also, the first item in the MRO is the type itself, which is supposed - // to be already checked by the caller. We skip it in the loop. - assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); - assert(PyType_GetSlot(type, Py_tp_token) != token); - - Py_ssize_t n = PyTuple_GET_SIZE(mro); - for (Py_ssize_t i = 1; i < n; i++) { - PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(mro, i)); - if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { - continue; - } - if (((PyHeapTypeObject*)base)->ht_token == token) { - return base; - } + if (result != NULL) { + *result = NULL; } - return NULL; -} -static int -check_base_by_token(PyTypeObject *type, void *token) { - // Chain the branches, which will be optimized exclusive here if (token == NULL) { PyErr_Format(PyExc_SystemError, "PyType_GetBaseByToken called with token=NULL"); return -1; } - else if (!PyType_Check(type)) { + if (!PyType_Check(type)) { PyErr_Format(PyExc_TypeError, "expected a type, got a '%T' object", type); return -1; } - else if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - return 0; - } - else if (((PyHeapTypeObject*)type)->ht_token == token) { - return 1; - } - else if (type->tp_mro != NULL) { - // This will not be inlined - return get_base_by_token_from_mro(type, token) ? 1 : 0; - } - else { - return get_base_by_token_recursive(type, token) ? 1 : 0; - } -} - -int -PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) -{ - if (result == NULL) { - // If the `result` is checked only once here, the subsequent - // branches will become trivial to optimize. - return check_base_by_token(type, token); - } - if (token == NULL || !PyType_Check(type)) { - *result = NULL; - return check_base_by_token(type, token); - } - // Chain the branches, which will be optimized exclusive here - PyTypeObject *base; if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // No static type has a heaptype superclass, // which is ensured by type_ready_mro(). - *result = NULL; return 0; } - else if (((PyHeapTypeObject*)type)->ht_token == token) { - *result = (PyTypeObject *)Py_NewRef(type); + if (((PyHeapTypeObject*)type)->ht_token == token) { +found: + if (result != NULL) { + *result = (PyTypeObject *)Py_NewRef(type); + } return 1; } - else if (type->tp_mro != NULL) { - // Expect this to be inlined - base = get_base_by_token_from_mro(type, token); - } - else { - base = get_base_by_token_recursive(type, token); - } - if (base != NULL) { - *result = (PyTypeObject *)Py_NewRef(base); - return 1; - } - else { - *result = NULL; + PyObject *mro = type->tp_mro; // No lookup, following PyType_IsSubtype() + if (mro == NULL) { + PyTypeObject *base; + base = get_base_by_token_recursive(lookup_tp_bases(type), token); + if (base != NULL) { + // Copying the given type can cause a slowdown, + // unlike the overwrite below. + type = base; + goto found; + } return 0; } + // mro_invoke() ensures that the type MRO cannot be empty. + assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t i = 1; i < n; i++) { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(mro, i); + if (_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE) + && ((PyHeapTypeObject*)base)->ht_token == token) { + type = base; + goto found; + } + } + return 0; } @@ -6023,7 +5966,7 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, int isbuiltin, int final) { assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - assert(_Py_IsImmortalLoose((PyObject *)type)); + assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); @@ -6100,7 +6043,7 @@ type_dealloc(PyObject *self) Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); #ifdef Py_GIL_DISABLED - _PyType_ReleaseId(et); + _PyObject_ReleaseUniqueId(et->unique_id); #endif et->ht_token = NULL; Py_TYPE(type)->tp_free((PyObject *)type); @@ -7416,18 +7359,7 @@ static PyObject * object___reduce_ex___impl(PyObject *self, int protocol) /*[clinic end generated code: output=2e157766f6b50094 input=f326b43fb8a4c5ff]*/ { -#define objreduce \ - (_Py_INTERP_CACHED_OBJECT(_PyInterpreterState_GET(), objreduce)) - PyObject *reduce, *res; - - if (objreduce == NULL) { - PyObject *dict = lookup_tp_dict(&PyBaseObject_Type); - objreduce = PyDict_GetItemWithError(dict, &_Py_ID(__reduce__)); - if (objreduce == NULL && PyErr_Occurred()) { - return NULL; - } - } - + PyObject *reduce; if (PyObject_GetOptionalAttr(self, &_Py_ID(__reduce__), &reduce) < 0) { return NULL; } @@ -7441,10 +7373,12 @@ object___reduce_ex___impl(PyObject *self, int protocol) Py_DECREF(reduce); return NULL; } - override = (clsreduce != objreduce); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + override = (clsreduce != _Py_INTERP_CACHED_OBJECT(interp, objreduce)); Py_DECREF(clsreduce); if (override) { - res = _PyObject_CallNoArgs(reduce); + PyObject *res = _PyObject_CallNoArgs(reduce); Py_DECREF(reduce); return res; } @@ -7453,7 +7387,6 @@ object___reduce_ex___impl(PyObject *self, int protocol) } return _common_reduce(self, protocol); -#undef objreduce } static PyObject * diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 3c96850589d378..91cc37c9a72636 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -151,7 +151,7 @@ constevaluator_clear(PyObject *self) } static PyObject * -constevaluator_repr(PyObject *self, PyObject *repr) +constevaluator_repr(PyObject *self) { PyObject *value = ((constevaluatorobject *)self)->value; return PyUnicode_FromFormat("", value); @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 3) { // SOURCE + if (format == 3) { // STRING PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; @@ -242,7 +242,8 @@ static PyType_Slot constevaluator_slots[] = { PyType_Spec constevaluator_spec = { .name = "_typing._ConstEvaluator", .basicsize = sizeof(constevaluatorobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_DISALLOW_INSTANTIATION, .slots = constevaluator_slots, }; @@ -372,10 +373,10 @@ caller(void) if (f == NULL) { Py_RETURN_NONE; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *r = PyFunction_GetModule(PyStackRef_AsPyObjectBorrow(f->f_funcobj)); if (!r) { PyErr_Clear(); Py_RETURN_NONE; @@ -1798,6 +1799,24 @@ _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) return (PyObject *)typevartuple_alloc(v, NULL, NULL); } +static PyObject * +get_type_param_default(PyThreadState *ts, PyObject *typeparam) { + // Does not modify refcount of existing objects. + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + return typevar_default((typevarobject *)typeparam, NULL); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + return paramspec_default((paramspecobject *)typeparam, NULL); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + return typevartuple_default((typevartupleobject *)typeparam, NULL); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } +} + static void typealias_dealloc(PyObject *self) { @@ -1905,6 +1924,65 @@ static PyGetSetDef typealias_getset[] = { {0} }; +static PyObject * +typealias_check_type_params(PyObject *type_params, int *err) { + // Can return type_params or NULL without exception set. + // Does not change the reference count of type_params, + // sets `*err` to 1 when error happens and sets an exception, + // otherwise `*err` is set to 0. + *err = 0; + if (type_params == NULL) { + return NULL; + } + + assert(PyTuple_Check(type_params)); + Py_ssize_t length = PyTuple_GET_SIZE(type_params); + if (!length) { // 0-length tuples are the same as `NULL`. + return NULL; + } + + PyThreadState *ts = _PyThreadState_GET(); + int default_seen = 0; + for (Py_ssize_t index = 0; index < length; index++) { + PyObject *type_param = PyTuple_GET_ITEM(type_params, index); + PyObject *dflt = get_type_param_default(ts, type_param); + if (dflt == NULL) { + *err = 1; + return NULL; + } + if (dflt == &_Py_NoDefaultStruct) { + if (default_seen) { + *err = 1; + PyErr_Format(PyExc_TypeError, + "non-default type parameter '%R' " + "follows default type parameter", + type_param); + return NULL; + } + } else { + default_seen = 1; + Py_DECREF(dflt); + } + } + + return type_params; +} + +static PyObject * +typelias_convert_type_params(PyObject *type_params) +{ + if ( + type_params == NULL + || Py_IsNone(type_params) + || (PyTuple_Check(type_params) && PyTuple_GET_SIZE(type_params) == 0) + ) { + return NULL; + } + else { + return type_params; + } +} + static typealiasobject * typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, PyObject *value, PyObject *module) @@ -1914,7 +1992,7 @@ typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, return NULL; } ta->name = Py_NewRef(name); - ta->type_params = Py_IsNone(type_params) ? NULL : Py_XNewRef(type_params); + ta->type_params = Py_XNewRef(type_params); ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); ta->module = Py_XNewRef(module); @@ -1992,11 +2070,18 @@ typealias_new_impl(PyTypeObject *type, PyObject *name, PyObject *value, PyErr_SetString(PyExc_TypeError, "type_params must be a tuple"); return NULL; } + + int err = 0; + PyObject *checked_params = typealias_check_type_params(type_params, &err); + if (err) { + return NULL; + } + PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *ta = (PyObject *)typealias_alloc(name, type_params, NULL, value, + PyObject *ta = (PyObject *)typealias_alloc(name, checked_params, NULL, value, module); Py_DECREF(module); return ta; @@ -2062,7 +2147,7 @@ _Py_make_typealias(PyThreadState* unused, PyObject *args) assert(PyTuple_GET_SIZE(args) == 3); PyObject *name = PyTuple_GET_ITEM(args, 0); assert(PyUnicode_Check(name)); - PyObject *type_params = PyTuple_GET_ITEM(args, 1); + PyObject *type_params = typelias_convert_type_params(PyTuple_GET_ITEM(args, 1)); PyObject *compute_value = PyTuple_GET_ITEM(args, 2); assert(PyFunction_Check(compute_value)); return (PyObject *)typealias_alloc(name, type_params, compute_value, NULL, NULL); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2494c989544ca0..b94a74c2c688a9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -46,6 +46,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_codecs.h" // _PyCodec_Lookup() #include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST #include "pycore_format.h" // F_LJUST +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_FormatWriter() @@ -261,7 +262,6 @@ _PyUnicode_InternedSize_Immortal(void) } static Py_hash_t unicode_hash(PyObject *); -static int unicode_compare_eq(PyObject *, PyObject *); static Py_uhash_t hashtable_unicode_hash(const void *key) @@ -275,20 +275,44 @@ hashtable_unicode_compare(const void *key1, const void *key2) PyObject *obj1 = (PyObject *)key1; PyObject *obj2 = (PyObject *)key2; if (obj1 != NULL && obj2 != NULL) { - return unicode_compare_eq(obj1, obj2); + return unicode_eq(obj1, obj2); } else { return obj1 == obj2; } } +/* Return true if this interpreter should share the main interpreter's + intern_dict. That's important for interpreters which load basic + single-phase init extension modules (m_size == -1). There could be interned + immortal strings that are shared between interpreters, due to the + PyDict_Update(mdict, m_copy) call in import_find_extension(). + + It's not safe to deallocate those strings until all interpreters that + potentially use them are freed. By storing them in the main interpreter, we + ensure they get freed after all other interpreters are freed. +*/ +static bool +has_shared_intern_dict(PyInterpreterState *interp) +{ + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + return interp != main_interp && interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC; +} + static int init_interned_dict(PyInterpreterState *interp) { assert(get_interned_dict(interp) == NULL); - PyObject *interned = interned = PyDict_New(); - if (interned == NULL) { - return -1; + PyObject *interned; + if (has_shared_intern_dict(interp)) { + interned = get_interned_dict(_PyInterpreterState_Main()); + Py_INCREF(interned); + } + else { + interned = PyDict_New(); + if (interned == NULL) { + return -1; + } } _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = interned; return 0; @@ -299,7 +323,10 @@ clear_interned_dict(PyInterpreterState *interp) { PyObject *interned = get_interned_dict(interp); if (interned != NULL) { - PyDict_Clear(interned); + if (!has_shared_intern_dict(interp)) { + // only clear if the dict belongs to this interpreter + PyDict_Clear(interned); + } Py_DECREF(interned); _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL; } @@ -2694,11 +2721,6 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, #define F_SIZE 3 #define F_PTRDIFF 4 #define F_INTMAX 5 -static const char * const formats[] = {"%d", "%ld", "%lld", "%zd", "%td", "%jd"}; -static const char * const formats_o[] = {"%o", "%lo", "%llo", "%zo", "%to", "%jo"}; -static const char * const formats_u[] = {"%u", "%lu", "%llu", "%zu", "%tu", "%ju"}; -static const char * const formats_x[] = {"%x", "%lx", "%llx", "%zx", "%tx", "%jx"}; -static const char * const formats_X[] = {"%X", "%lX", "%llX", "%zX", "%tX", "%jX"}; static const char* unicode_fromformat_arg(_PyUnicodeWriter *writer, @@ -2840,47 +2862,44 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': { - /* used by sprintf */ char buffer[MAX_INTMAX_CHARS]; - const char *fmt = NULL; - switch (*f) { - case 'o': fmt = formats_o[sizemod]; break; - case 'u': fmt = formats_u[sizemod]; break; - case 'x': fmt = formats_x[sizemod]; break; - case 'X': fmt = formats_X[sizemod]; break; - default: fmt = formats[sizemod]; break; - } - int issigned = (*f == 'd' || *f == 'i'); + + // Fill buffer using sprinf, with one of many possible format + // strings, like "%llX" for `long long` in hexadecimal. + // The type/size is in `sizemod`; the format is in `*f`. + + // Use macros with nested switches to keep the sprintf format strings + // as compile-time literals, avoiding warnings and maybe allowing + // optimizations. + + // `SPRINT` macro does one sprintf + // Example usage: SPRINT("l", "X", unsigned long) expands to + // sprintf(buffer, "%" "l" "X", va_arg(*vargs, unsigned long)) + #define SPRINT(SIZE_SPEC, FMT_CHAR, TYPE) \ + sprintf(buffer, "%" SIZE_SPEC FMT_CHAR, va_arg(*vargs, TYPE)) + + // One inner switch to handle all format variants + #define DO_SPRINTS(SIZE_SPEC, SIGNED_TYPE, UNSIGNED_TYPE) \ + switch (*f) { \ + case 'o': len = SPRINT(SIZE_SPEC, "o", UNSIGNED_TYPE); break; \ + case 'u': len = SPRINT(SIZE_SPEC, "u", UNSIGNED_TYPE); break; \ + case 'x': len = SPRINT(SIZE_SPEC, "x", UNSIGNED_TYPE); break; \ + case 'X': len = SPRINT(SIZE_SPEC, "X", UNSIGNED_TYPE); break; \ + default: len = SPRINT(SIZE_SPEC, "d", SIGNED_TYPE); break; \ + } + + // Outer switch to handle all the sizes/types switch (sizemod) { - case F_LONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long)); - break; - case F_LONGLONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long long)); - break; - case F_SIZE: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, Py_ssize_t)) : - sprintf(buffer, fmt, va_arg(*vargs, size_t)); - break; - case F_PTRDIFF: - len = sprintf(buffer, fmt, va_arg(*vargs, ptrdiff_t)); - break; - case F_INTMAX: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, intmax_t)) : - sprintf(buffer, fmt, va_arg(*vargs, uintmax_t)); - break; - default: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, int)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned int)); - break; + case F_LONG: DO_SPRINTS("l", long, unsigned long); break; + case F_LONGLONG: DO_SPRINTS("ll", long long, unsigned long long); break; + case F_SIZE: DO_SPRINTS("z", Py_ssize_t, size_t); break; + case F_PTRDIFF: DO_SPRINTS("t", ptrdiff_t, ptrdiff_t); break; + case F_INTMAX: DO_SPRINTS("j", intmax_t, uintmax_t); break; + default: DO_SPRINTS("", int, unsigned int); break; } + #undef SPRINT + #undef DO_SPRINTS + assert(len >= 0); int sign = (buffer[0] == '-'); @@ -10976,26 +10995,6 @@ unicode_compare(PyObject *str1, PyObject *str2) #undef COMPARE } -static int -unicode_compare_eq(PyObject *str1, PyObject *str2) -{ - int kind; - const void *data1, *data2; - Py_ssize_t len; - int cmp; - - len = PyUnicode_GET_LENGTH(str1); - if (PyUnicode_GET_LENGTH(str2) != len) - return 0; - kind = PyUnicode_KIND(str1); - if (PyUnicode_KIND(str2) != kind) - return 0; - data1 = PyUnicode_DATA(str1); - data2 = PyUnicode_DATA(str2); - - cmp = memcmp(data1, data2, len * kind); - return (cmp == 0); -} int _PyUnicode_Equal(PyObject *str1, PyObject *str2) @@ -11005,7 +11004,25 @@ _PyUnicode_Equal(PyObject *str1, PyObject *str2) if (str1 == str2) { return 1; } - return unicode_compare_eq(str1, str2); + return unicode_eq(str1, str2); +} + + +int +PyUnicode_Equal(PyObject *str1, PyObject *str2) +{ + if (!PyUnicode_Check(str1)) { + PyErr_Format(PyExc_TypeError, + "first argument must be str, not %T", str1); + return -1; + } + if (!PyUnicode_Check(str2)) { + PyErr_Format(PyExc_TypeError, + "second argument must be str, not %T", str2); + return -1; + } + + return _PyUnicode_Equal(str1, str2); } @@ -11203,7 +11220,7 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) return 0; } - return unicode_compare_eq(left, right_uni); + return unicode_eq(left, right_uni); } PyObject * @@ -11231,7 +11248,7 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } } else if (op == Py_EQ || op == Py_NE) { - result = unicode_compare_eq(left, right); + result = unicode_eq(left, right); result ^= (op == Py_NE); return PyBool_FromLong(result); } @@ -11241,12 +11258,6 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } } -int -_PyUnicode_EQ(PyObject *aa, PyObject *bb) -{ - return unicode_eq(aa, bb); -} - int PyUnicode_Contains(PyObject *str, PyObject *substr) { @@ -13426,9 +13437,13 @@ PyUnicodeWriter_Create(Py_ssize_t length) } const size_t size = sizeof(_PyUnicodeWriter); - PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + PyUnicodeWriter *pub_writer; + pub_writer = _Py_FREELIST_POP_MEM(unicode_writers); if (pub_writer == NULL) { - return (PyUnicodeWriter *)PyErr_NoMemory(); + pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == NULL) { + return (PyUnicodeWriter *)PyErr_NoMemory(); + } } _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; @@ -13445,8 +13460,11 @@ PyUnicodeWriter_Create(Py_ssize_t length) void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) { + if (writer == NULL) { + return; + } _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); - PyMem_Free(writer); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); } @@ -13622,6 +13640,10 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) int PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) { + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + PyObject *str = PyObject_Str(obj); if (str == NULL) { return -1; @@ -13636,6 +13658,10 @@ PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) { + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + PyObject *repr = PyObject_Repr(obj); if (repr == NULL) { return -1; @@ -13860,7 +13886,7 @@ PyUnicodeWriter_Finish(PyUnicodeWriter *writer) { PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); assert(((_PyUnicodeWriter*)writer)->buffer == NULL); - PyMem_Free(writer); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); return str; } @@ -15418,6 +15444,10 @@ _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **p) assert(*p); } +#ifdef Py_TRACE_REFS +extern void _Py_NormalizeImmortalReference(PyObject *); +#endif + static void immortalize_interned(PyObject *s) { @@ -15433,6 +15463,10 @@ immortalize_interned(PyObject *s) #endif _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL; _Py_SetImmortal(s); +#ifdef Py_TRACE_REFS + /* Make sure the ref is associated with the right interpreter. */ + _Py_NormalizeImmortalReference(s); +#endif } static /* non-null */ PyObject* @@ -15626,6 +15660,13 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); + if (has_shared_intern_dict(interp)) { + // the dict doesn't belong to this interpreter, skip the debug + // checks on it and just clear the pointer to it + clear_interned_dict(interp); + return; + } + #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -16134,8 +16175,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) { struct _Py_unicode_state *state = &interp->unicode; - // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() - assert(get_interned_dict(interp) == NULL); + if (!has_shared_intern_dict(interp)) { + // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() + assert(get_interned_dict(interp) == NULL); + } _PyUnicode_FiniEncodings(&state->fs_codec); diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 61f05514a48023..9e3da1c3394d5b 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -104,9 +104,11 @@ clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) // Clear the weakref and its callback static void -clear_weakref(PyWeakReference *self) +clear_weakref(PyObject *op) { + PyWeakReference *self = _PyWeakref_CAST(op); PyObject *callback = NULL; + // self->wr_object may be Py_None if the GC cleared the weakref, so lock // using the pointer in the weakref. LOCK_WEAKREFS_FOR_WR(self); @@ -139,22 +141,24 @@ static void weakref_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); - clear_weakref((PyWeakReference *) self); + clear_weakref(self); Py_TYPE(self)->tp_free(self); } static int -gc_traverse(PyWeakReference *self, visitproc visit, void *arg) +gc_traverse(PyObject *op, visitproc visit, void *arg) { + PyWeakReference *self = _PyWeakref_CAST(op); Py_VISIT(self->wr_callback); return 0; } static int -gc_clear(PyWeakReference *self) +gc_clear(PyObject *op) { + PyWeakReference *self = _PyWeakref_CAST(op); PyObject *callback; // The world is stopped during GC in free-threaded builds. It's safe to // call this without holding the lock. @@ -198,8 +202,9 @@ weakref_hash_lock_held(PyWeakReference *self) } static Py_hash_t -weakref_hash(PyWeakReference *self) +weakref_hash(PyObject *op) { + PyWeakReference *self = _PyWeakref_CAST(op); Py_hash_t hash; Py_BEGIN_CRITICAL_SECTION(self); hash = weakref_hash_lock_held(self); @@ -499,11 +504,11 @@ _PyWeakref_RefType = { .tp_vectorcall_offset = offsetof(PyWeakReference, vectorcall), .tp_call = PyVectorcall_Call, .tp_repr = weakref_repr, - .tp_hash = (hashfunc)weakref_hash, + .tp_hash = weakref_hash, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_BASETYPE, - .tp_traverse = (traverseproc)gc_traverse, - .tp_clear = (inquiry)gc_clear, + .tp_traverse = gc_traverse, + .tp_clear = gc_clear, .tp_richcompare = weakref_richcompare, .tp_methods = weakref_methods, .tp_members = weakref_members, @@ -687,7 +692,7 @@ proxy_bool(PyObject *proxy) } static void -proxy_dealloc(PyWeakReference *self) +proxy_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); clear_weakref(self); @@ -850,7 +855,7 @@ _PyWeakref_ProxyType = { sizeof(PyWeakReference), 0, /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + proxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -868,8 +873,8 @@ _PyWeakref_ProxyType = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gc_traverse, /* tp_traverse */ - (inquiry)gc_clear, /* tp_clear */ + gc_traverse, /* tp_traverse */ + gc_clear, /* tp_clear */ proxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ proxy_iter, /* tp_iter */ @@ -885,7 +890,7 @@ _PyWeakref_CallableProxyType = { sizeof(PyWeakReference), 0, /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + proxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -902,8 +907,8 @@ _PyWeakref_CallableProxyType = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gc_traverse, /* tp_traverse */ - (inquiry)gc_clear, /* tp_clear */ + gc_traverse, /* tp_traverse */ + gc_clear, /* tp_clear */ proxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ proxy_iter, /* tp_iter */ diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index 503f3193e2803e..010f5fe5646630 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -169,9 +169,9 @@ WIN32 is still required for the locale module. #endif /* MS_WIN64 */ /* set the version macros for the windows headers */ -/* Python 3.9+ requires Windows 8 or greater */ -#define Py_WINVER 0x0602 /* _WIN32_WINNT_WIN8 */ -#define Py_NTDDI NTDDI_WIN8 +/* Python 3.13+ requires Windows 10 or greater */ +#define Py_WINVER 0x0A00 /* _WIN32_WINNT_WIN10 */ +#define Py_NTDDI NTDDI_WIN10 /* We only set these values when building Python - we don't want to force these values on extensions, as that will affect the prototypes and diff --git a/PC/python3dll.c b/PC/python3dll.c index 6b8208ab90bd95..9296474617e115 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -717,6 +717,7 @@ EXPORT_FUNC(PyUnicode_DecodeUTF8Stateful) EXPORT_FUNC(PyUnicode_EncodeCodePage) EXPORT_FUNC(PyUnicode_EncodeFSDefault) EXPORT_FUNC(PyUnicode_EncodeLocale) +EXPORT_FUNC(PyUnicode_Equal) EXPORT_FUNC(PyUnicode_EqualToUTF8) EXPORT_FUNC(PyUnicode_EqualToUTF8AndSize) EXPORT_FUNC(PyUnicode_Find) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 743e6e2a66a8f1..a3c2d32c454e04 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -268,7 +268,7 @@ - + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 0887a47917a931..91b1d75fb8df5e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -467,7 +467,7 @@ Source Files - + Source Files diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index a1409ecf043d2d..846e027e10c7fa 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,6 +97,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index e27e3171e1e6aa..57be2e2fc5b950 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,6 +12,7 @@ + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 137c94789e1809..dfacd1d1e788d4 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -56,8 +56,8 @@ if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.15 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.45.3.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.14.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.14.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-1.3.1 @@ -78,7 +78,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.15 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.14.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 19b982db7f5b87..3b33c6bf6bb91d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -304,13 +304,13 @@ - + @@ -657,7 +657,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 23f2e9c8bc0eb7..ee2930b10439a9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -831,9 +831,6 @@ Include\internal - - Include\internal - Include\internal @@ -846,6 +843,9 @@ Include\internal + + Include\internal + Include\internal\mimalloc @@ -1499,7 +1499,7 @@ Python - + Python diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 865e294d260a49..693fcee5f90ce2 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -195,7 +195,7 @@ _sqlite3 Homepage: https://www.sqlite.org/ _tkinter - Wraps version 8.6.14 of the Tk windowing system, which is downloaded + Wraps version 8.6.15 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 83c38c993d5754..b4cb401609d409 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.14.0 + 8.6.15.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 1972c606827cdb..cb21777f566189 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1541,7 +1541,7 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } if (bytes_found) { - PyObject* res = PyBytes_FromString(""); + PyObject* res = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); /* Bytes literals never get a kind, but just for consistency since they are represented as Constant nodes, we'll mirror @@ -1615,7 +1615,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } /* build folded list */ - _PyUnicodeWriter writer; current_pos = 0; for (i = 0; i < n_flattened_elements; i++) { expr_ty elem = asdl_seq_GET(flattened, i); @@ -1635,14 +1634,17 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, "abc" u"abc" -> "abcabc" */ PyObject *kind = elem->v.Constant.kind; - _PyUnicodeWriter_Init(&writer); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } expr_ty last_elem = elem; for (j = i; j < n_flattened_elements; j++) { expr_ty current_elem = asdl_seq_GET(flattened, j); if (current_elem->kind == Constant_kind) { - if (_PyUnicodeWriter_WriteStr( - &writer, current_elem->v.Constant.value)) { - _PyUnicodeWriter_Dealloc(&writer); + if (PyUnicodeWriter_WriteStr(writer, + current_elem->v.Constant.value)) { + PyUnicodeWriter_Discard(writer); return NULL; } last_elem = current_elem; @@ -1652,9 +1654,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } i = j - 1; - PyObject *concat_str = _PyUnicodeWriter_Finish(&writer); + PyObject *concat_str = PyUnicodeWriter_Finish(writer); if (concat_str == NULL) { - _PyUnicodeWriter_Dealloc(&writer); return NULL; } if (_PyArena_AddPyObject(p->arena, concat_str) < 0) { diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index fac9a7740a1fe6..32eac3afafa5d5 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -843,8 +843,9 @@ def visitModule(self, mod): } AST_object; static void -ast_dealloc(AST_object *self) +ast_dealloc(PyObject *op) { + AST_object *self = (AST_object*)op; /* bpo-31095: UnTrack is needed before calling any callbacks */ PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); @@ -856,16 +857,18 @@ def visitModule(self, mod): } static int -ast_traverse(AST_object *self, visitproc visit, void *arg) +ast_traverse(PyObject *op, visitproc visit, void *arg) { + AST_object *self = (AST_object*)op; Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -ast_clear(AST_object *self) +ast_clear(PyObject *op) { + AST_object *self = (AST_object*)op; Py_CLEAR(self->dict); return 0; } @@ -1605,7 +1608,6 @@ def visitModule(self, mod): if (!value_repr) { Py_DECREF(name); - Py_DECREF(value); goto error; } @@ -1651,9 +1653,9 @@ def visitModule(self, mod): } static PyObject * -ast_repr(AST_object *self) +ast_repr(PyObject *self) { - return ast_repr_max_depth(self, 3); + return ast_repr_max_depth((AST_object*)self, 3); } static PyType_Slot AST_type_slots[] = { @@ -1847,8 +1849,9 @@ def visitModule(self, mod): self.file.write(textwrap.dedent(''' static int - init_types(struct ast_state *state) + init_types(void *arg) { + struct ast_state *state = arg; if (init_identifiers(state) < 0) { return -1; } @@ -2239,8 +2242,6 @@ def generate_ast_fini(module_state, f): for s in module_state: f.write(" Py_CLEAR(state->" + s + ');\n') f.write(textwrap.dedent(""" - Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); - state->finalized = 1; state->once = (_PyOnceFlag){0}; } @@ -2296,7 +2297,7 @@ def generate_module_def(mod, metadata, f, internal_h): }; // Forward declaration - static int init_types(struct ast_state *state); + static int init_types(void *arg); static struct ast_state* get_ast_state(void) diff --git a/Parser/parser.c b/Parser/parser.c index e5567d0f63f03b..9ff58ab7e7bfd5 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -338,156 +338,156 @@ static char *soft_keywords[] = { #define invalid_factor_type 1251 #define invalid_type_params_type 1252 #define _loop0_1_type 1253 -#define _loop0_2_type 1254 -#define _loop1_3_type 1255 -#define _loop0_5_type 1256 -#define _gather_4_type 1257 +#define _loop1_2_type 1254 +#define _loop0_3_type 1255 +#define _gather_4_type 1256 +#define _tmp_5_type 1257 #define _tmp_6_type 1258 #define _tmp_7_type 1259 #define _tmp_8_type 1260 #define _tmp_9_type 1261 #define _tmp_10_type 1262 #define _tmp_11_type 1263 -#define _tmp_12_type 1264 +#define _loop1_12_type 1264 #define _tmp_13_type 1265 -#define _loop1_14_type 1266 -#define _tmp_15_type 1267 +#define _loop0_14_type 1266 +#define _gather_15_type 1267 #define _tmp_16_type 1268 #define _tmp_17_type 1269 -#define _loop0_19_type 1270 -#define _gather_18_type 1271 -#define _loop0_21_type 1272 -#define _gather_20_type 1273 +#define _loop0_18_type 1270 +#define _loop1_19_type 1271 +#define _loop0_20_type 1272 +#define _gather_21_type 1273 #define _tmp_22_type 1274 -#define _tmp_23_type 1275 -#define _loop0_24_type 1276 +#define _loop0_23_type 1275 +#define _gather_24_type 1276 #define _loop1_25_type 1277 -#define _loop0_27_type 1278 -#define _gather_26_type 1279 -#define _tmp_28_type 1280 -#define _loop0_30_type 1281 -#define _gather_29_type 1282 -#define _tmp_31_type 1283 -#define _loop1_32_type 1284 -#define _tmp_33_type 1285 -#define _tmp_34_type 1286 -#define _tmp_35_type 1287 -#define _loop0_36_type 1288 -#define _loop0_37_type 1289 -#define _loop0_38_type 1290 +#define _tmp_26_type 1278 +#define _tmp_27_type 1279 +#define _loop0_28_type 1280 +#define _loop0_29_type 1281 +#define _loop1_30_type 1282 +#define _loop1_31_type 1283 +#define _loop0_32_type 1284 +#define _loop1_33_type 1285 +#define _loop0_34_type 1286 +#define _gather_35_type 1287 +#define _tmp_36_type 1288 +#define _loop1_37_type 1289 +#define _loop1_38_type 1290 #define _loop1_39_type 1291 #define _loop0_40_type 1292 -#define _loop1_41_type 1293 -#define _loop1_42_type 1294 -#define _loop1_43_type 1295 +#define _gather_41_type 1293 +#define _tmp_42_type 1294 +#define _tmp_43_type 1295 #define _loop0_44_type 1296 -#define _loop1_45_type 1297 +#define _gather_45_type 1297 #define _loop0_46_type 1298 -#define _loop1_47_type 1299 -#define _loop0_48_type 1300 +#define _gather_47_type 1299 +#define _tmp_48_type 1300 #define _loop0_49_type 1301 -#define _loop1_50_type 1302 -#define _loop0_52_type 1303 -#define _gather_51_type 1304 -#define _loop0_54_type 1305 -#define _gather_53_type 1306 -#define _loop0_56_type 1307 -#define _gather_55_type 1308 -#define _loop0_58_type 1309 -#define _gather_57_type 1310 -#define _tmp_59_type 1311 +#define _gather_50_type 1302 +#define _loop0_51_type 1303 +#define _gather_52_type 1304 +#define _loop0_53_type 1305 +#define _gather_54_type 1306 +#define _loop1_55_type 1307 +#define _loop1_56_type 1308 +#define _loop0_57_type 1309 +#define _gather_58_type 1310 +#define _loop1_59_type 1311 #define _loop1_60_type 1312 #define _loop1_61_type 1313 #define _tmp_62_type 1314 -#define _tmp_63_type 1315 -#define _loop1_64_type 1316 -#define _loop0_66_type 1317 -#define _gather_65_type 1318 +#define _loop0_63_type 1315 +#define _gather_64_type 1316 +#define _tmp_65_type 1317 +#define _tmp_66_type 1318 #define _tmp_67_type 1319 #define _tmp_68_type 1320 #define _tmp_69_type 1321 #define _tmp_70_type 1322 -#define _loop0_72_type 1323 -#define _gather_71_type 1324 -#define _loop0_74_type 1325 -#define _gather_73_type 1326 -#define _tmp_75_type 1327 -#define _loop0_77_type 1328 -#define _gather_76_type 1329 -#define _loop0_79_type 1330 -#define _gather_78_type 1331 -#define _loop0_81_type 1332 -#define _gather_80_type 1333 -#define _loop1_82_type 1334 +#define _loop0_71_type 1323 +#define _loop0_72_type 1324 +#define _loop1_73_type 1325 +#define _loop1_74_type 1326 +#define _loop0_75_type 1327 +#define _loop1_76_type 1328 +#define _loop0_77_type 1329 +#define _loop0_78_type 1330 +#define _loop1_79_type 1331 +#define _tmp_80_type 1332 +#define _loop0_81_type 1333 +#define _gather_82_type 1334 #define _loop1_83_type 1335 -#define _loop0_85_type 1336 -#define _gather_84_type 1337 -#define _loop1_86_type 1338 -#define _loop1_87_type 1339 -#define _loop1_88_type 1340 -#define _tmp_89_type 1341 -#define _loop0_91_type 1342 -#define _gather_90_type 1343 -#define _tmp_92_type 1344 -#define _tmp_93_type 1345 -#define _tmp_94_type 1346 -#define _tmp_95_type 1347 -#define _tmp_96_type 1348 +#define _loop0_84_type 1336 +#define _tmp_85_type 1337 +#define _loop0_86_type 1338 +#define _gather_87_type 1339 +#define _tmp_88_type 1340 +#define _loop0_89_type 1341 +#define _gather_90_type 1342 +#define _loop0_91_type 1343 +#define _gather_92_type 1344 +#define _loop0_93_type 1345 +#define _loop0_94_type 1346 +#define _gather_95_type 1347 +#define _loop1_96_type 1348 #define _tmp_97_type 1349 #define _loop0_98_type 1350 -#define _loop0_99_type 1351 +#define _gather_99_type 1351 #define _loop0_100_type 1352 -#define _loop1_101_type 1353 -#define _loop0_102_type 1354 -#define _loop1_103_type 1355 -#define _loop1_104_type 1356 -#define _loop1_105_type 1357 -#define _loop0_106_type 1358 -#define _loop1_107_type 1359 -#define _loop0_108_type 1360 -#define _loop1_109_type 1361 -#define _loop0_110_type 1362 -#define _loop1_111_type 1363 -#define _loop0_112_type 1364 -#define _loop0_113_type 1365 -#define _loop1_114_type 1366 -#define _tmp_115_type 1367 -#define _loop0_117_type 1368 -#define _gather_116_type 1369 -#define _loop1_118_type 1370 -#define _loop0_119_type 1371 -#define _loop0_120_type 1372 +#define _gather_101_type 1353 +#define _tmp_102_type 1354 +#define _tmp_103_type 1355 +#define _loop0_104_type 1356 +#define _gather_105_type 1357 +#define _tmp_106_type 1358 +#define _tmp_107_type 1359 +#define _tmp_108_type 1360 +#define _tmp_109_type 1361 +#define _tmp_110_type 1362 +#define _tmp_111_type 1363 +#define _tmp_112_type 1364 +#define _tmp_113_type 1365 +#define _tmp_114_type 1366 +#define _loop0_115_type 1367 +#define _loop0_116_type 1368 +#define _tmp_117_type 1369 +#define _tmp_118_type 1370 +#define _tmp_119_type 1371 +#define _tmp_120_type 1372 #define _tmp_121_type 1373 -#define _loop0_123_type 1374 -#define _gather_122_type 1375 +#define _tmp_122_type 1374 +#define _tmp_123_type 1375 #define _tmp_124_type 1376 -#define _loop0_126_type 1377 -#define _gather_125_type 1378 -#define _loop0_128_type 1379 -#define _gather_127_type 1380 -#define _loop0_130_type 1381 -#define _gather_129_type 1382 -#define _loop0_132_type 1383 -#define _gather_131_type 1384 -#define _loop0_133_type 1385 -#define _loop0_135_type 1386 -#define _gather_134_type 1387 -#define _loop1_136_type 1388 -#define _tmp_137_type 1389 -#define _loop0_139_type 1390 -#define _gather_138_type 1391 -#define _loop0_141_type 1392 -#define _gather_140_type 1393 -#define _loop0_143_type 1394 -#define _gather_142_type 1395 -#define _loop0_145_type 1396 -#define _gather_144_type 1397 -#define _loop0_147_type 1398 -#define _gather_146_type 1399 +#define _tmp_125_type 1377 +#define _loop0_126_type 1378 +#define _gather_127_type 1379 +#define _tmp_128_type 1380 +#define _tmp_129_type 1381 +#define _tmp_130_type 1382 +#define _tmp_131_type 1383 +#define _loop0_132_type 1384 +#define _gather_133_type 1385 +#define _loop0_134_type 1386 +#define _gather_135_type 1387 +#define _loop0_136_type 1388 +#define _gather_137_type 1389 +#define _tmp_138_type 1390 +#define _loop0_139_type 1391 +#define _tmp_140_type 1392 +#define _tmp_141_type 1393 +#define _tmp_142_type 1394 +#define _tmp_143_type 1395 +#define _tmp_144_type 1396 +#define _tmp_145_type 1397 +#define _tmp_146_type 1398 +#define _tmp_147_type 1399 #define _tmp_148_type 1400 #define _tmp_149_type 1401 -#define _loop0_151_type 1402 -#define _gather_150_type 1403 +#define _tmp_150_type 1402 +#define _tmp_151_type 1403 #define _tmp_152_type 1404 #define _tmp_153_type 1405 #define _tmp_154_type 1406 @@ -498,128 +498,14 @@ static char *soft_keywords[] = { #define _tmp_159_type 1411 #define _tmp_160_type 1412 #define _tmp_161_type 1413 -#define _loop0_162_type 1414 -#define _loop0_163_type 1415 -#define _loop0_164_type 1416 -#define _tmp_165_type 1417 +#define _tmp_162_type 1414 +#define _tmp_163_type 1415 +#define _tmp_164_type 1416 +#define _loop0_165_type 1417 #define _tmp_166_type 1418 #define _tmp_167_type 1419 #define _tmp_168_type 1420 -#define _loop0_169_type 1421 -#define _loop0_170_type 1422 -#define _loop0_171_type 1423 -#define _loop1_172_type 1424 -#define _tmp_173_type 1425 -#define _loop0_174_type 1426 -#define _tmp_175_type 1427 -#define _loop0_176_type 1428 -#define _loop1_177_type 1429 -#define _tmp_178_type 1430 -#define _tmp_179_type 1431 -#define _tmp_180_type 1432 -#define _loop0_181_type 1433 -#define _tmp_182_type 1434 -#define _tmp_183_type 1435 -#define _loop1_184_type 1436 -#define _tmp_185_type 1437 -#define _loop0_186_type 1438 -#define _loop0_187_type 1439 -#define _loop0_188_type 1440 -#define _loop0_190_type 1441 -#define _gather_189_type 1442 -#define _tmp_191_type 1443 -#define _loop0_192_type 1444 -#define _tmp_193_type 1445 -#define _loop0_194_type 1446 -#define _loop1_195_type 1447 -#define _loop1_196_type 1448 -#define _tmp_197_type 1449 -#define _tmp_198_type 1450 -#define _loop0_199_type 1451 -#define _tmp_200_type 1452 -#define _tmp_201_type 1453 -#define _tmp_202_type 1454 -#define _tmp_203_type 1455 -#define _loop0_205_type 1456 -#define _gather_204_type 1457 -#define _loop0_207_type 1458 -#define _gather_206_type 1459 -#define _loop0_209_type 1460 -#define _gather_208_type 1461 -#define _loop0_211_type 1462 -#define _gather_210_type 1463 -#define _loop0_213_type 1464 -#define _gather_212_type 1465 -#define _tmp_214_type 1466 -#define _loop0_215_type 1467 -#define _loop1_216_type 1468 -#define _tmp_217_type 1469 -#define _loop0_218_type 1470 -#define _loop1_219_type 1471 -#define _tmp_220_type 1472 -#define _tmp_221_type 1473 -#define _tmp_222_type 1474 -#define _tmp_223_type 1475 -#define _tmp_224_type 1476 -#define _tmp_225_type 1477 -#define _tmp_226_type 1478 -#define _tmp_227_type 1479 -#define _tmp_228_type 1480 -#define _tmp_229_type 1481 -#define _tmp_230_type 1482 -#define _tmp_231_type 1483 -#define _tmp_232_type 1484 -#define _loop0_234_type 1485 -#define _gather_233_type 1486 -#define _tmp_235_type 1487 -#define _tmp_236_type 1488 -#define _tmp_237_type 1489 -#define _tmp_238_type 1490 -#define _tmp_239_type 1491 -#define _tmp_240_type 1492 -#define _tmp_241_type 1493 -#define _loop0_242_type 1494 -#define _tmp_243_type 1495 -#define _tmp_244_type 1496 -#define _tmp_245_type 1497 -#define _tmp_246_type 1498 -#define _tmp_247_type 1499 -#define _tmp_248_type 1500 -#define _tmp_249_type 1501 -#define _tmp_250_type 1502 -#define _tmp_251_type 1503 -#define _tmp_252_type 1504 -#define _tmp_253_type 1505 -#define _tmp_254_type 1506 -#define _tmp_255_type 1507 -#define _tmp_256_type 1508 -#define _tmp_257_type 1509 -#define _tmp_258_type 1510 -#define _tmp_259_type 1511 -#define _tmp_260_type 1512 -#define _tmp_261_type 1513 -#define _tmp_262_type 1514 -#define _tmp_263_type 1515 -#define _tmp_264_type 1516 -#define _tmp_265_type 1517 -#define _tmp_266_type 1518 -#define _tmp_267_type 1519 -#define _loop0_268_type 1520 -#define _tmp_269_type 1521 -#define _tmp_270_type 1522 -#define _tmp_271_type 1523 -#define _tmp_272_type 1524 -#define _tmp_273_type 1525 -#define _tmp_274_type 1526 -#define _loop0_276_type 1527 -#define _gather_275_type 1528 -#define _tmp_277_type 1529 -#define _tmp_278_type 1530 -#define _tmp_279_type 1531 -#define _tmp_280_type 1532 -#define _tmp_281_type 1533 -#define _tmp_282_type 1534 -#define _tmp_283_type 1535 +#define _tmp_169_type 1421 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -875,156 +761,156 @@ static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); static void *invalid_type_params_rule(Parser *p); static asdl_seq *_loop0_1_rule(Parser *p); -static asdl_seq *_loop0_2_rule(Parser *p); -static asdl_seq *_loop1_3_rule(Parser *p); -static asdl_seq *_loop0_5_rule(Parser *p); +static asdl_seq *_loop1_2_rule(Parser *p); +static asdl_seq *_loop0_3_rule(Parser *p); static asdl_seq *_gather_4_rule(Parser *p); +static void *_tmp_5_rule(Parser *p); static void *_tmp_6_rule(Parser *p); static void *_tmp_7_rule(Parser *p); static void *_tmp_8_rule(Parser *p); static void *_tmp_9_rule(Parser *p); static void *_tmp_10_rule(Parser *p); static void *_tmp_11_rule(Parser *p); -static void *_tmp_12_rule(Parser *p); +static asdl_seq *_loop1_12_rule(Parser *p); static void *_tmp_13_rule(Parser *p); -static asdl_seq *_loop1_14_rule(Parser *p); -static void *_tmp_15_rule(Parser *p); +static asdl_seq *_loop0_14_rule(Parser *p); +static asdl_seq *_gather_15_rule(Parser *p); static void *_tmp_16_rule(Parser *p); static void *_tmp_17_rule(Parser *p); -static asdl_seq *_loop0_19_rule(Parser *p); -static asdl_seq *_gather_18_rule(Parser *p); -static asdl_seq *_loop0_21_rule(Parser *p); -static asdl_seq *_gather_20_rule(Parser *p); +static asdl_seq *_loop0_18_rule(Parser *p); +static asdl_seq *_loop1_19_rule(Parser *p); +static asdl_seq *_loop0_20_rule(Parser *p); +static asdl_seq *_gather_21_rule(Parser *p); static void *_tmp_22_rule(Parser *p); -static void *_tmp_23_rule(Parser *p); -static asdl_seq *_loop0_24_rule(Parser *p); +static asdl_seq *_loop0_23_rule(Parser *p); +static asdl_seq *_gather_24_rule(Parser *p); static asdl_seq *_loop1_25_rule(Parser *p); -static asdl_seq *_loop0_27_rule(Parser *p); -static asdl_seq *_gather_26_rule(Parser *p); -static void *_tmp_28_rule(Parser *p); -static asdl_seq *_loop0_30_rule(Parser *p); -static asdl_seq *_gather_29_rule(Parser *p); -static void *_tmp_31_rule(Parser *p); -static asdl_seq *_loop1_32_rule(Parser *p); -static void *_tmp_33_rule(Parser *p); -static void *_tmp_34_rule(Parser *p); -static void *_tmp_35_rule(Parser *p); -static asdl_seq *_loop0_36_rule(Parser *p); -static asdl_seq *_loop0_37_rule(Parser *p); -static asdl_seq *_loop0_38_rule(Parser *p); +static void *_tmp_26_rule(Parser *p); +static void *_tmp_27_rule(Parser *p); +static asdl_seq *_loop0_28_rule(Parser *p); +static asdl_seq *_loop0_29_rule(Parser *p); +static asdl_seq *_loop1_30_rule(Parser *p); +static asdl_seq *_loop1_31_rule(Parser *p); +static asdl_seq *_loop0_32_rule(Parser *p); +static asdl_seq *_loop1_33_rule(Parser *p); +static asdl_seq *_loop0_34_rule(Parser *p); +static asdl_seq *_gather_35_rule(Parser *p); +static void *_tmp_36_rule(Parser *p); +static asdl_seq *_loop1_37_rule(Parser *p); +static asdl_seq *_loop1_38_rule(Parser *p); static asdl_seq *_loop1_39_rule(Parser *p); static asdl_seq *_loop0_40_rule(Parser *p); -static asdl_seq *_loop1_41_rule(Parser *p); -static asdl_seq *_loop1_42_rule(Parser *p); -static asdl_seq *_loop1_43_rule(Parser *p); +static asdl_seq *_gather_41_rule(Parser *p); +static void *_tmp_42_rule(Parser *p); +static void *_tmp_43_rule(Parser *p); static asdl_seq *_loop0_44_rule(Parser *p); -static asdl_seq *_loop1_45_rule(Parser *p); +static asdl_seq *_gather_45_rule(Parser *p); static asdl_seq *_loop0_46_rule(Parser *p); -static asdl_seq *_loop1_47_rule(Parser *p); -static asdl_seq *_loop0_48_rule(Parser *p); +static asdl_seq *_gather_47_rule(Parser *p); +static void *_tmp_48_rule(Parser *p); static asdl_seq *_loop0_49_rule(Parser *p); -static asdl_seq *_loop1_50_rule(Parser *p); -static asdl_seq *_loop0_52_rule(Parser *p); -static asdl_seq *_gather_51_rule(Parser *p); -static asdl_seq *_loop0_54_rule(Parser *p); -static asdl_seq *_gather_53_rule(Parser *p); -static asdl_seq *_loop0_56_rule(Parser *p); -static asdl_seq *_gather_55_rule(Parser *p); -static asdl_seq *_loop0_58_rule(Parser *p); -static asdl_seq *_gather_57_rule(Parser *p); -static void *_tmp_59_rule(Parser *p); +static asdl_seq *_gather_50_rule(Parser *p); +static asdl_seq *_loop0_51_rule(Parser *p); +static asdl_seq *_gather_52_rule(Parser *p); +static asdl_seq *_loop0_53_rule(Parser *p); +static asdl_seq *_gather_54_rule(Parser *p); +static asdl_seq *_loop1_55_rule(Parser *p); +static asdl_seq *_loop1_56_rule(Parser *p); +static asdl_seq *_loop0_57_rule(Parser *p); +static asdl_seq *_gather_58_rule(Parser *p); +static asdl_seq *_loop1_59_rule(Parser *p); static asdl_seq *_loop1_60_rule(Parser *p); static asdl_seq *_loop1_61_rule(Parser *p); static void *_tmp_62_rule(Parser *p); -static void *_tmp_63_rule(Parser *p); -static asdl_seq *_loop1_64_rule(Parser *p); -static asdl_seq *_loop0_66_rule(Parser *p); -static asdl_seq *_gather_65_rule(Parser *p); +static asdl_seq *_loop0_63_rule(Parser *p); +static asdl_seq *_gather_64_rule(Parser *p); +static void *_tmp_65_rule(Parser *p); +static void *_tmp_66_rule(Parser *p); static void *_tmp_67_rule(Parser *p); static void *_tmp_68_rule(Parser *p); static void *_tmp_69_rule(Parser *p); static void *_tmp_70_rule(Parser *p); +static asdl_seq *_loop0_71_rule(Parser *p); static asdl_seq *_loop0_72_rule(Parser *p); -static asdl_seq *_gather_71_rule(Parser *p); -static asdl_seq *_loop0_74_rule(Parser *p); -static asdl_seq *_gather_73_rule(Parser *p); -static void *_tmp_75_rule(Parser *p); +static asdl_seq *_loop1_73_rule(Parser *p); +static asdl_seq *_loop1_74_rule(Parser *p); +static asdl_seq *_loop0_75_rule(Parser *p); +static asdl_seq *_loop1_76_rule(Parser *p); static asdl_seq *_loop0_77_rule(Parser *p); -static asdl_seq *_gather_76_rule(Parser *p); -static asdl_seq *_loop0_79_rule(Parser *p); -static asdl_seq *_gather_78_rule(Parser *p); +static asdl_seq *_loop0_78_rule(Parser *p); +static asdl_seq *_loop1_79_rule(Parser *p); +static void *_tmp_80_rule(Parser *p); static asdl_seq *_loop0_81_rule(Parser *p); -static asdl_seq *_gather_80_rule(Parser *p); -static asdl_seq *_loop1_82_rule(Parser *p); +static asdl_seq *_gather_82_rule(Parser *p); static asdl_seq *_loop1_83_rule(Parser *p); -static asdl_seq *_loop0_85_rule(Parser *p); -static asdl_seq *_gather_84_rule(Parser *p); -static asdl_seq *_loop1_86_rule(Parser *p); -static asdl_seq *_loop1_87_rule(Parser *p); -static asdl_seq *_loop1_88_rule(Parser *p); -static void *_tmp_89_rule(Parser *p); -static asdl_seq *_loop0_91_rule(Parser *p); +static asdl_seq *_loop0_84_rule(Parser *p); +static void *_tmp_85_rule(Parser *p); +static asdl_seq *_loop0_86_rule(Parser *p); +static asdl_seq *_gather_87_rule(Parser *p); +static void *_tmp_88_rule(Parser *p); +static asdl_seq *_loop0_89_rule(Parser *p); static asdl_seq *_gather_90_rule(Parser *p); -static void *_tmp_92_rule(Parser *p); -static void *_tmp_93_rule(Parser *p); -static void *_tmp_94_rule(Parser *p); -static void *_tmp_95_rule(Parser *p); -static void *_tmp_96_rule(Parser *p); +static asdl_seq *_loop0_91_rule(Parser *p); +static asdl_seq *_gather_92_rule(Parser *p); +static asdl_seq *_loop0_93_rule(Parser *p); +static asdl_seq *_loop0_94_rule(Parser *p); +static asdl_seq *_gather_95_rule(Parser *p); +static asdl_seq *_loop1_96_rule(Parser *p); static void *_tmp_97_rule(Parser *p); static asdl_seq *_loop0_98_rule(Parser *p); -static asdl_seq *_loop0_99_rule(Parser *p); +static asdl_seq *_gather_99_rule(Parser *p); static asdl_seq *_loop0_100_rule(Parser *p); -static asdl_seq *_loop1_101_rule(Parser *p); -static asdl_seq *_loop0_102_rule(Parser *p); -static asdl_seq *_loop1_103_rule(Parser *p); -static asdl_seq *_loop1_104_rule(Parser *p); -static asdl_seq *_loop1_105_rule(Parser *p); -static asdl_seq *_loop0_106_rule(Parser *p); -static asdl_seq *_loop1_107_rule(Parser *p); -static asdl_seq *_loop0_108_rule(Parser *p); -static asdl_seq *_loop1_109_rule(Parser *p); -static asdl_seq *_loop0_110_rule(Parser *p); -static asdl_seq *_loop1_111_rule(Parser *p); -static asdl_seq *_loop0_112_rule(Parser *p); -static asdl_seq *_loop0_113_rule(Parser *p); -static asdl_seq *_loop1_114_rule(Parser *p); -static void *_tmp_115_rule(Parser *p); -static asdl_seq *_loop0_117_rule(Parser *p); -static asdl_seq *_gather_116_rule(Parser *p); -static asdl_seq *_loop1_118_rule(Parser *p); -static asdl_seq *_loop0_119_rule(Parser *p); -static asdl_seq *_loop0_120_rule(Parser *p); +static asdl_seq *_gather_101_rule(Parser *p); +static void *_tmp_102_rule(Parser *p); +static void *_tmp_103_rule(Parser *p); +static asdl_seq *_loop0_104_rule(Parser *p); +static asdl_seq *_gather_105_rule(Parser *p); +static void *_tmp_106_rule(Parser *p); +static void *_tmp_107_rule(Parser *p); +static void *_tmp_108_rule(Parser *p); +static void *_tmp_109_rule(Parser *p); +static void *_tmp_110_rule(Parser *p); +static void *_tmp_111_rule(Parser *p); +static void *_tmp_112_rule(Parser *p); +static void *_tmp_113_rule(Parser *p); +static void *_tmp_114_rule(Parser *p); +static asdl_seq *_loop0_115_rule(Parser *p); +static asdl_seq *_loop0_116_rule(Parser *p); +static void *_tmp_117_rule(Parser *p); +static void *_tmp_118_rule(Parser *p); +static void *_tmp_119_rule(Parser *p); +static void *_tmp_120_rule(Parser *p); static void *_tmp_121_rule(Parser *p); -static asdl_seq *_loop0_123_rule(Parser *p); -static asdl_seq *_gather_122_rule(Parser *p); +static void *_tmp_122_rule(Parser *p); +static void *_tmp_123_rule(Parser *p); static void *_tmp_124_rule(Parser *p); +static void *_tmp_125_rule(Parser *p); static asdl_seq *_loop0_126_rule(Parser *p); -static asdl_seq *_gather_125_rule(Parser *p); -static asdl_seq *_loop0_128_rule(Parser *p); static asdl_seq *_gather_127_rule(Parser *p); -static asdl_seq *_loop0_130_rule(Parser *p); -static asdl_seq *_gather_129_rule(Parser *p); +static void *_tmp_128_rule(Parser *p); +static void *_tmp_129_rule(Parser *p); +static void *_tmp_130_rule(Parser *p); +static void *_tmp_131_rule(Parser *p); static asdl_seq *_loop0_132_rule(Parser *p); -static asdl_seq *_gather_131_rule(Parser *p); -static asdl_seq *_loop0_133_rule(Parser *p); -static asdl_seq *_loop0_135_rule(Parser *p); -static asdl_seq *_gather_134_rule(Parser *p); -static asdl_seq *_loop1_136_rule(Parser *p); -static void *_tmp_137_rule(Parser *p); +static asdl_seq *_gather_133_rule(Parser *p); +static asdl_seq *_loop0_134_rule(Parser *p); +static asdl_seq *_gather_135_rule(Parser *p); +static asdl_seq *_loop0_136_rule(Parser *p); +static asdl_seq *_gather_137_rule(Parser *p); +static void *_tmp_138_rule(Parser *p); static asdl_seq *_loop0_139_rule(Parser *p); -static asdl_seq *_gather_138_rule(Parser *p); -static asdl_seq *_loop0_141_rule(Parser *p); -static asdl_seq *_gather_140_rule(Parser *p); -static asdl_seq *_loop0_143_rule(Parser *p); -static asdl_seq *_gather_142_rule(Parser *p); -static asdl_seq *_loop0_145_rule(Parser *p); -static asdl_seq *_gather_144_rule(Parser *p); -static asdl_seq *_loop0_147_rule(Parser *p); -static asdl_seq *_gather_146_rule(Parser *p); +static void *_tmp_140_rule(Parser *p); +static void *_tmp_141_rule(Parser *p); +static void *_tmp_142_rule(Parser *p); +static void *_tmp_143_rule(Parser *p); +static void *_tmp_144_rule(Parser *p); +static void *_tmp_145_rule(Parser *p); +static void *_tmp_146_rule(Parser *p); +static void *_tmp_147_rule(Parser *p); static void *_tmp_148_rule(Parser *p); static void *_tmp_149_rule(Parser *p); -static asdl_seq *_loop0_151_rule(Parser *p); -static asdl_seq *_gather_150_rule(Parser *p); +static void *_tmp_150_rule(Parser *p); +static void *_tmp_151_rule(Parser *p); static void *_tmp_152_rule(Parser *p); static void *_tmp_153_rule(Parser *p); static void *_tmp_154_rule(Parser *p); @@ -1035,128 +921,14 @@ static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); static void *_tmp_160_rule(Parser *p); static void *_tmp_161_rule(Parser *p); -static asdl_seq *_loop0_162_rule(Parser *p); -static asdl_seq *_loop0_163_rule(Parser *p); -static asdl_seq *_loop0_164_rule(Parser *p); -static void *_tmp_165_rule(Parser *p); +static void *_tmp_162_rule(Parser *p); +static void *_tmp_163_rule(Parser *p); +static void *_tmp_164_rule(Parser *p); +static asdl_seq *_loop0_165_rule(Parser *p); static void *_tmp_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); -static asdl_seq *_loop0_169_rule(Parser *p); -static asdl_seq *_loop0_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); -static asdl_seq *_loop1_172_rule(Parser *p); -static void *_tmp_173_rule(Parser *p); -static asdl_seq *_loop0_174_rule(Parser *p); -static void *_tmp_175_rule(Parser *p); -static asdl_seq *_loop0_176_rule(Parser *p); -static asdl_seq *_loop1_177_rule(Parser *p); -static void *_tmp_178_rule(Parser *p); -static void *_tmp_179_rule(Parser *p); -static void *_tmp_180_rule(Parser *p); -static asdl_seq *_loop0_181_rule(Parser *p); -static void *_tmp_182_rule(Parser *p); -static void *_tmp_183_rule(Parser *p); -static asdl_seq *_loop1_184_rule(Parser *p); -static void *_tmp_185_rule(Parser *p); -static asdl_seq *_loop0_186_rule(Parser *p); -static asdl_seq *_loop0_187_rule(Parser *p); -static asdl_seq *_loop0_188_rule(Parser *p); -static asdl_seq *_loop0_190_rule(Parser *p); -static asdl_seq *_gather_189_rule(Parser *p); -static void *_tmp_191_rule(Parser *p); -static asdl_seq *_loop0_192_rule(Parser *p); -static void *_tmp_193_rule(Parser *p); -static asdl_seq *_loop0_194_rule(Parser *p); -static asdl_seq *_loop1_195_rule(Parser *p); -static asdl_seq *_loop1_196_rule(Parser *p); -static void *_tmp_197_rule(Parser *p); -static void *_tmp_198_rule(Parser *p); -static asdl_seq *_loop0_199_rule(Parser *p); -static void *_tmp_200_rule(Parser *p); -static void *_tmp_201_rule(Parser *p); -static void *_tmp_202_rule(Parser *p); -static void *_tmp_203_rule(Parser *p); -static asdl_seq *_loop0_205_rule(Parser *p); -static asdl_seq *_gather_204_rule(Parser *p); -static asdl_seq *_loop0_207_rule(Parser *p); -static asdl_seq *_gather_206_rule(Parser *p); -static asdl_seq *_loop0_209_rule(Parser *p); -static asdl_seq *_gather_208_rule(Parser *p); -static asdl_seq *_loop0_211_rule(Parser *p); -static asdl_seq *_gather_210_rule(Parser *p); -static asdl_seq *_loop0_213_rule(Parser *p); -static asdl_seq *_gather_212_rule(Parser *p); -static void *_tmp_214_rule(Parser *p); -static asdl_seq *_loop0_215_rule(Parser *p); -static asdl_seq *_loop1_216_rule(Parser *p); -static void *_tmp_217_rule(Parser *p); -static asdl_seq *_loop0_218_rule(Parser *p); -static asdl_seq *_loop1_219_rule(Parser *p); -static void *_tmp_220_rule(Parser *p); -static void *_tmp_221_rule(Parser *p); -static void *_tmp_222_rule(Parser *p); -static void *_tmp_223_rule(Parser *p); -static void *_tmp_224_rule(Parser *p); -static void *_tmp_225_rule(Parser *p); -static void *_tmp_226_rule(Parser *p); -static void *_tmp_227_rule(Parser *p); -static void *_tmp_228_rule(Parser *p); -static void *_tmp_229_rule(Parser *p); -static void *_tmp_230_rule(Parser *p); -static void *_tmp_231_rule(Parser *p); -static void *_tmp_232_rule(Parser *p); -static asdl_seq *_loop0_234_rule(Parser *p); -static asdl_seq *_gather_233_rule(Parser *p); -static void *_tmp_235_rule(Parser *p); -static void *_tmp_236_rule(Parser *p); -static void *_tmp_237_rule(Parser *p); -static void *_tmp_238_rule(Parser *p); -static void *_tmp_239_rule(Parser *p); -static void *_tmp_240_rule(Parser *p); -static void *_tmp_241_rule(Parser *p); -static asdl_seq *_loop0_242_rule(Parser *p); -static void *_tmp_243_rule(Parser *p); -static void *_tmp_244_rule(Parser *p); -static void *_tmp_245_rule(Parser *p); -static void *_tmp_246_rule(Parser *p); -static void *_tmp_247_rule(Parser *p); -static void *_tmp_248_rule(Parser *p); -static void *_tmp_249_rule(Parser *p); -static void *_tmp_250_rule(Parser *p); -static void *_tmp_251_rule(Parser *p); -static void *_tmp_252_rule(Parser *p); -static void *_tmp_253_rule(Parser *p); -static void *_tmp_254_rule(Parser *p); -static void *_tmp_255_rule(Parser *p); -static void *_tmp_256_rule(Parser *p); -static void *_tmp_257_rule(Parser *p); -static void *_tmp_258_rule(Parser *p); -static void *_tmp_259_rule(Parser *p); -static void *_tmp_260_rule(Parser *p); -static void *_tmp_261_rule(Parser *p); -static void *_tmp_262_rule(Parser *p); -static void *_tmp_263_rule(Parser *p); -static void *_tmp_264_rule(Parser *p); -static void *_tmp_265_rule(Parser *p); -static void *_tmp_266_rule(Parser *p); -static void *_tmp_267_rule(Parser *p); -static asdl_seq *_loop0_268_rule(Parser *p); -static void *_tmp_269_rule(Parser *p); -static void *_tmp_270_rule(Parser *p); -static void *_tmp_271_rule(Parser *p); -static void *_tmp_272_rule(Parser *p); -static void *_tmp_273_rule(Parser *p); -static void *_tmp_274_rule(Parser *p); -static asdl_seq *_loop0_276_rule(Parser *p); -static asdl_seq *_gather_275_rule(Parser *p); -static void *_tmp_277_rule(Parser *p); -static void *_tmp_278_rule(Parser *p); -static void *_tmp_279_rule(Parser *p); -static void *_tmp_280_rule(Parser *p); -static void *_tmp_281_rule(Parser *p); -static void *_tmp_282_rule(Parser *p); -static void *_tmp_283_rule(Parser *p); +static void *_tmp_169_rule(Parser *p); // file: statements? $ @@ -1319,7 +1091,7 @@ func_type_rule(Parser *p) Token * _literal; Token * _literal_1; Token * _literal_2; - asdl_seq * _loop0_2_var; + asdl_seq * _loop0_1_var; void *a; expr_ty b; Token * endmarker_var; @@ -1334,7 +1106,7 @@ func_type_rule(Parser *p) && (b = expression_rule(p)) // expression && - (_loop0_2_var = _loop0_2_rule(p)) // NEWLINE* + (_loop0_1_var = _loop0_1_rule(p)) // NEWLINE* && (endmarker_var = _PyPegen_expect_token(p, ENDMARKER)) // token='ENDMARKER' ) @@ -1379,7 +1151,7 @@ statements_rule(Parser *p) D(fprintf(stderr, "%*c> statements[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "statement+")); asdl_seq * a; if ( - (a = _loop1_3_rule(p)) // statement+ + (a = _loop1_2_rule(p)) // statement+ ) { D(fprintf(stderr, "%*c+ statements[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "statement+")); @@ -1820,7 +1592,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); stmt_ty import_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_6_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_5_rule, p) && (import_stmt_var = import_stmt_rule(p)) // import_stmt ) @@ -2094,7 +1866,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('def' | '@' | 'async') function_def")); stmt_ty function_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_7_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_6_rule, p) && (function_def_var = function_def_rule(p)) // function_def ) @@ -2136,7 +1908,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('class' | '@') class_def")); stmt_ty class_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_8_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_7_rule, p) && (class_def_var = class_def_rule(p)) // class_def ) @@ -2157,7 +1929,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('with' | 'async') with_stmt")); stmt_ty with_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_9_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_8_rule, p) && (with_stmt_var = with_stmt_rule(p)) // with_stmt ) @@ -2178,7 +1950,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('for' | 'async') for_stmt")); stmt_ty for_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_10_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_9_rule, p) && (for_stmt_var = for_stmt_rule(p)) // for_stmt ) @@ -2261,8 +2033,8 @@ compound_stmt_rule(Parser *p) // assignment: // | NAME ':' expression ['=' annotated_rhs] // | ('(' single_target ')' | single_subscript_attribute_target) ':' expression ['=' annotated_rhs] -// | ((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT? -// | single_target augassign ~ (yield_expr | star_expressions) +// | ((star_targets '='))+ annotated_rhs !'=' TYPE_COMMENT? +// | single_target augassign ~ annotated_rhs // | invalid_assignment static stmt_ty assignment_rule(Parser *p) @@ -2302,7 +2074,7 @@ assignment_rule(Parser *p) && (b = expression_rule(p)) // expression && - (c = _tmp_11_rule(p), !p->error_indicator) // ['=' annotated_rhs] + (c = _tmp_10_rule(p), !p->error_indicator) // ['=' annotated_rhs] ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ':' expression ['=' annotated_rhs]")); @@ -2338,13 +2110,13 @@ assignment_rule(Parser *p) expr_ty b; void *c; if ( - (a = _tmp_12_rule(p)) // '(' single_target ')' | single_subscript_attribute_target + (a = _tmp_11_rule(p)) // '(' single_target ')' | single_subscript_attribute_target && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (b = expression_rule(p)) // expression && - (c = _tmp_13_rule(p), !p->error_indicator) // ['=' annotated_rhs] + (c = _tmp_10_rule(p), !p->error_indicator) // ['=' annotated_rhs] ) { D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('(' single_target ')' | single_subscript_attribute_target) ':' expression ['=' annotated_rhs]")); @@ -2369,26 +2141,26 @@ assignment_rule(Parser *p) D(fprintf(stderr, "%*c%s assignment[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('(' single_target ')' | single_subscript_attribute_target) ':' expression ['=' annotated_rhs]")); } - { // ((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT? + { // ((star_targets '='))+ annotated_rhs !'=' TYPE_COMMENT? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT?")); + D(fprintf(stderr, "%*c> assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))+ annotated_rhs !'=' TYPE_COMMENT?")); asdl_expr_seq* a; - void *b; + expr_ty b; void *tc; if ( - (a = (asdl_expr_seq*)_loop1_14_rule(p)) // ((star_targets '='))+ + (a = (asdl_expr_seq*)_loop1_12_rule(p)) // ((star_targets '='))+ && - (b = _tmp_15_rule(p)) // yield_expr | star_expressions + (b = annotated_rhs_rule(p)) // annotated_rhs && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' && (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? ) { - D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT?")); + D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((star_targets '='))+ annotated_rhs !'=' TYPE_COMMENT?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -2408,18 +2180,18 @@ assignment_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s assignment[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT?")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((star_targets '='))+ annotated_rhs !'=' TYPE_COMMENT?")); } - { // single_target augassign ~ (yield_expr | star_expressions) + { // single_target augassign ~ annotated_rhs if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_target augassign ~ (yield_expr | star_expressions)")); + D(fprintf(stderr, "%*c> assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_target augassign ~ annotated_rhs")); int _cut_var = 0; expr_ty a; AugOperator* b; - void *c; + expr_ty c; if ( (a = single_target_rule(p)) // single_target && @@ -2427,10 +2199,10 @@ assignment_rule(Parser *p) && (_cut_var = 1) && - (c = _tmp_16_rule(p)) // yield_expr | star_expressions + (c = annotated_rhs_rule(p)) // annotated_rhs ) { - D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_target augassign ~ (yield_expr | star_expressions)")); + D(fprintf(stderr, "%*c+ assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_target augassign ~ annotated_rhs")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -2450,7 +2222,7 @@ assignment_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s assignment[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "single_target augassign ~ (yield_expr | star_expressions)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "single_target augassign ~ annotated_rhs")); if (_cut_var) { p->level--; return NULL; @@ -2982,7 +2754,7 @@ raise_stmt_rule(Parser *p) && (a = expression_rule(p)) // expression && - (b = _tmp_17_rule(p), !p->error_indicator) // ['from' expression] + (b = _tmp_13_rule(p), !p->error_indicator) // ['from' expression] ) { D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); @@ -3079,7 +2851,7 @@ global_stmt_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 526)) // token='global' && - (a = (asdl_expr_seq*)_gather_18_rule(p)) // ','.NAME+ + (a = (asdl_expr_seq*)_gather_15_rule(p)) // ','.NAME+ ) { D(fprintf(stderr, "%*c+ global_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'global' ','.NAME+")); @@ -3143,7 +2915,7 @@ nonlocal_stmt_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 527)) // token='nonlocal' && - (a = (asdl_expr_seq*)_gather_20_rule(p)) // ','.NAME+ + (a = (asdl_expr_seq*)_gather_15_rule(p)) // ','.NAME+ ) { D(fprintf(stderr, "%*c+ nonlocal_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'nonlocal' ','.NAME+")); @@ -3209,7 +2981,7 @@ del_stmt_rule(Parser *p) && (a = del_targets_rule(p)) // del_targets && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_22_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_16_rule, p) ) { D(fprintf(stderr, "%*c+ del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' del_targets &(';' | NEWLINE)")); @@ -3356,7 +3128,7 @@ assert_stmt_rule(Parser *p) && (a = expression_rule(p)) // expression && - (b = _tmp_23_rule(p), !p->error_indicator) // [',' expression] + (b = _tmp_17_rule(p), !p->error_indicator) // [',' expression] ) { D(fprintf(stderr, "%*c+ assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression [',' expression]")); @@ -3565,7 +3337,7 @@ import_from_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && - (a = _loop0_24_rule(p)) // (('.' | '...'))* + (a = _loop0_18_rule(p)) // (('.' | '...'))* && (b = dotted_name_rule(p)) // dotted_name && @@ -3609,7 +3381,7 @@ import_from_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && - (a = _loop1_25_rule(p)) // (('.' | '...'))+ + (a = _loop1_19_rule(p)) // (('.' | '...'))+ && (_keyword_1 = _PyPegen_expect_token(p, 622)) // token='import' && @@ -3804,7 +3576,7 @@ import_from_as_names_rule(Parser *p) D(fprintf(stderr, "%*c> import_from_as_names[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); asdl_alias_seq* a; if ( - (a = (asdl_alias_seq*)_gather_26_rule(p)) // ','.import_from_as_name+ + (a = (asdl_alias_seq*)_gather_21_rule(p)) // ','.import_from_as_name+ ) { D(fprintf(stderr, "%*c+ import_from_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); @@ -3859,7 +3631,7 @@ import_from_as_name_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_28_rule(p), !p->error_indicator) // ['as' NAME] + (b = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ['as' NAME]")); @@ -3911,7 +3683,7 @@ dotted_as_names_rule(Parser *p) D(fprintf(stderr, "%*c> dotted_as_names[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); asdl_alias_seq* a; if ( - (a = (asdl_alias_seq*)_gather_29_rule(p)) // ','.dotted_as_name+ + (a = (asdl_alias_seq*)_gather_24_rule(p)) // ','.dotted_as_name+ ) { D(fprintf(stderr, "%*c+ dotted_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); @@ -3966,7 +3738,7 @@ dotted_as_name_rule(Parser *p) if ( (a = dotted_name_rule(p)) // dotted_name && - (b = _tmp_31_rule(p), !p->error_indicator) // ['as' NAME] + (b = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name ['as' NAME]")); @@ -4217,7 +3989,7 @@ decorators_rule(Parser *p) D(fprintf(stderr, "%*c> decorators[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_32_rule(p)) // (('@' named_expression NEWLINE))+ + (a = (asdl_expr_seq*)_loop1_25_rule(p)) // (('@' named_expression NEWLINE))+ ) { D(fprintf(stderr, "%*c+ decorators[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); @@ -4366,7 +4138,7 @@ class_def_raw_rule(Parser *p) && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (b = _tmp_33_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (b = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4539,7 +4311,7 @@ function_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_34_rule(p), !p->error_indicator) // ['->' expression] + (a = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4602,7 +4374,7 @@ function_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_35_rule(p), !p->error_indicator) // ['->' expression] + (a = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4727,9 +4499,9 @@ parameters_rule(Parser *p) if ( (a = slash_no_default_rule(p)) // slash_no_default && - (b = (asdl_arg_seq*)_loop0_36_rule(p)) // param_no_default* + (b = (asdl_arg_seq*)_loop0_28_rule(p)) // param_no_default* && - (c = _loop0_37_rule(p)) // param_with_default* + (c = _loop0_29_rule(p)) // param_with_default* && (d = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4759,7 +4531,7 @@ parameters_rule(Parser *p) if ( (a = slash_with_default_rule(p)) // slash_with_default && - (b = _loop0_38_rule(p)) // param_with_default* + (b = _loop0_29_rule(p)) // param_with_default* && (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4787,9 +4559,9 @@ parameters_rule(Parser *p) asdl_seq * b; void *c; if ( - (a = (asdl_arg_seq*)_loop1_39_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ && - (b = _loop0_40_rule(p)) // param_with_default* + (b = _loop0_29_rule(p)) // param_with_default* && (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4816,7 +4588,7 @@ parameters_rule(Parser *p) asdl_seq * a; void *b; if ( - (a = _loop1_41_rule(p)) // param_with_default+ + (a = _loop1_31_rule(p)) // param_with_default+ && (b = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4887,7 +4659,7 @@ slash_no_default_rule(Parser *p) Token * _literal_1; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_42_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -4916,7 +4688,7 @@ slash_no_default_rule(Parser *p) Token * _literal; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_43_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -4968,9 +4740,9 @@ slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_44_rule(p)) // param_no_default* + (a = _loop0_28_rule(p)) // param_no_default* && - (b = _loop1_45_rule(p)) // param_with_default+ + (b = _loop1_31_rule(p)) // param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5000,9 +4772,9 @@ slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_46_rule(p)) // param_no_default* + (a = _loop0_28_rule(p)) // param_no_default* && - (b = _loop1_47_rule(p)) // param_with_default+ + (b = _loop1_31_rule(p)) // param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5080,7 +4852,7 @@ star_etc_rule(Parser *p) && (a = param_no_default_rule(p)) // param_no_default && - (b = _loop0_48_rule(p)) // param_maybe_default* + (b = _loop0_32_rule(p)) // param_maybe_default* && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5113,7 +4885,7 @@ star_etc_rule(Parser *p) && (a = param_no_default_star_annotation_rule(p)) // param_no_default_star_annotation && - (b = _loop0_49_rule(p)) // param_maybe_default* + (b = _loop0_32_rule(p)) // param_maybe_default* && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5146,7 +4918,7 @@ star_etc_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _loop1_50_rule(p)) // param_maybe_default+ + (b = _loop1_33_rule(p)) // param_maybe_default+ && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -6574,7 +6346,7 @@ with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = (asdl_withitem_seq*)_gather_51_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -6623,7 +6395,7 @@ with_stmt_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && - (a = (asdl_withitem_seq*)_gather_53_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6676,7 +6448,7 @@ with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = (asdl_withitem_seq*)_gather_55_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -6726,7 +6498,7 @@ with_stmt_rule(Parser *p) && (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='with' && - (a = (asdl_withitem_seq*)_gather_57_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6814,7 +6586,7 @@ with_item_rule(Parser *p) && (t = star_target_rule(p)) // star_target && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_59_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) ) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' star_target &(',' | ')' | ':')")); @@ -6985,7 +6757,7 @@ try_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (ex = (asdl_excepthandler_seq*)_loop1_60_rule(p)) // except_block+ + (ex = (asdl_excepthandler_seq*)_loop1_37_rule(p)) // except_block+ && (el = else_block_rule(p), !p->error_indicator) // else_block? && @@ -7033,7 +6805,7 @@ try_stmt_rule(Parser *p) && (b = block_rule(p)) // block && - (ex = (asdl_excepthandler_seq*)_loop1_61_rule(p)) // except_star_block+ + (ex = (asdl_excepthandler_seq*)_loop1_38_rule(p)) // except_star_block+ && (el = else_block_rule(p), !p->error_indicator) // else_block? && @@ -7129,7 +6901,7 @@ except_block_rule(Parser *p) && (e = expression_rule(p)) // expression && - (t = _tmp_62_rule(p), !p->error_indicator) // ['as' NAME] + (t = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7285,7 +7057,7 @@ except_star_block_rule(Parser *p) && (e = expression_rule(p)) // expression && - (t = _tmp_63_rule(p), !p->error_indicator) // ['as' NAME] + (t = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7455,7 +7227,7 @@ match_stmt_rule(Parser *p) && (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' && - (cases = (asdl_match_case_seq*)_loop1_64_rule(p)) // case_block+ + (cases = (asdl_match_case_seq*)_loop1_39_rule(p)) // case_block+ && (dedent_var = _PyPegen_expect_token(p, DEDENT)) // token='DEDENT' ) @@ -7966,7 +7738,7 @@ or_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> or_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); asdl_pattern_seq* patterns; if ( - (patterns = (asdl_pattern_seq*)_gather_65_rule(p)) // '|'.closed_pattern+ + (patterns = (asdl_pattern_seq*)_gather_41_rule(p)) // '|'.closed_pattern+ ) { D(fprintf(stderr, "%*c+ or_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); @@ -8219,7 +7991,7 @@ literal_pattern_rule(Parser *p) if ( (value = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_67_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8453,7 +8225,7 @@ literal_expr_rule(Parser *p) if ( (signed_number_var = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_68_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -9053,7 +8825,7 @@ pattern_capture_target_rule(Parser *p) && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_69_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_43_rule, p) ) { D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); @@ -9168,7 +8940,7 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_70_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_43_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); @@ -9587,7 +9359,7 @@ maybe_sequence_pattern_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * patterns; if ( - (patterns = _gather_71_rule(p)) // ','.maybe_star_pattern+ + (patterns = _gather_45_rule(p)) // ','.maybe_star_pattern+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -9995,13 +9767,13 @@ items_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> items_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); - asdl_seq * _gather_73_var; + asdl_seq * _gather_47_var; if ( - (_gather_73_var = _gather_73_rule(p)) // ','.key_value_pattern+ + (_gather_47_var = _gather_47_rule(p)) // ','.key_value_pattern+ ) { D(fprintf(stderr, "%*c+ items_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); - _res = _gather_73_var; + _res = _gather_47_var; goto done; } p->mark = _mark; @@ -10037,7 +9809,7 @@ key_value_pattern_rule(Parser *p) void *key; pattern_ty pattern; if ( - (key = _tmp_75_rule(p)) // literal_expr | attr + (key = _tmp_48_rule(p)) // literal_expr | attr && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -10365,7 +10137,7 @@ positional_patterns_rule(Parser *p) D(fprintf(stderr, "%*c> positional_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.pattern+")); asdl_pattern_seq* args; if ( - (args = (asdl_pattern_seq*)_gather_76_rule(p)) // ','.pattern+ + (args = (asdl_pattern_seq*)_gather_50_rule(p)) // ','.pattern+ ) { D(fprintf(stderr, "%*c+ positional_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.pattern+")); @@ -10406,13 +10178,13 @@ keyword_patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> keyword_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - asdl_seq * _gather_78_var; + asdl_seq * _gather_52_var; if ( - (_gather_78_var = _gather_78_rule(p)) // ','.keyword_pattern+ + (_gather_52_var = _gather_52_rule(p)) // ','.keyword_pattern+ ) { D(fprintf(stderr, "%*c+ keyword_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - _res = _gather_78_var; + _res = _gather_52_var; goto done; } p->mark = _mark; @@ -10638,7 +10410,7 @@ type_param_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_type_param_seq* a; if ( - (a = (asdl_type_param_seq*)_gather_80_rule(p)) // ','.type_param+ + (a = (asdl_type_param_seq*)_gather_54_rule(p)) // ','.type_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11008,7 +10780,7 @@ expressions_rule(Parser *p) if ( (a = expression_rule(p)) // expression && - (b = _loop1_82_rule(p)) // ((',' expression))+ + (b = _loop1_55_rule(p)) // ((',' expression))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11396,7 +11168,7 @@ star_expressions_rule(Parser *p) if ( (a = star_expression_rule(p)) // star_expression && - (b = _loop1_83_rule(p)) // ((',' star_expression))+ + (b = _loop1_56_rule(p)) // ((',' star_expression))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11595,7 +11367,7 @@ star_named_expressions_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_84_rule(p)) // ','.star_named_expression+ + (a = (asdl_expr_seq*)_gather_58_rule(p)) // ','.star_named_expression+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11891,7 +11663,7 @@ disjunction_rule(Parser *p) if ( (a = conjunction_rule(p)) // conjunction && - (b = _loop1_86_rule(p)) // (('or' conjunction))+ + (b = _loop1_59_rule(p)) // (('or' conjunction))+ ) { D(fprintf(stderr, "%*c+ disjunction[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "conjunction (('or' conjunction))+")); @@ -11979,7 +11751,7 @@ conjunction_rule(Parser *p) if ( (a = inversion_rule(p)) // inversion && - (b = _loop1_87_rule(p)) // (('and' inversion))+ + (b = _loop1_60_rule(p)) // (('and' inversion))+ ) { D(fprintf(stderr, "%*c+ conjunction[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "inversion (('and' inversion))+")); @@ -12151,7 +11923,7 @@ comparison_rule(Parser *p) if ( (a = bitwise_or_rule(p)) // bitwise_or && - (b = _loop1_88_rule(p)) // compare_op_bitwise_or_pair+ + (b = _loop1_61_rule(p)) // compare_op_bitwise_or_pair+ ) { D(fprintf(stderr, "%*c+ comparison[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or compare_op_bitwise_or_pair+")); @@ -12485,10 +12257,10 @@ noteq_bitwise_or_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> noteq_bitwise_or[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('!=') bitwise_or")); - void *_tmp_89_var; + void *_tmp_62_var; expr_ty a; if ( - (_tmp_89_var = _tmp_89_rule(p)) // '!=' + (_tmp_62_var = _tmp_62_rule(p)) // '!=' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -14536,7 +14308,7 @@ slices_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_90_rule(p)) // ','.(slice | starred_expression)+ + (a = (asdl_expr_seq*)_gather_64_rule(p)) // ','.(slice | starred_expression)+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -14608,7 +14380,7 @@ slice_rule(Parser *p) && (b = expression_rule(p), !p->error_indicator) // expression? && - (c = _tmp_92_rule(p), !p->error_indicator) // [':' expression?] + (c = _tmp_65_rule(p), !p->error_indicator) // [':' expression?] ) { D(fprintf(stderr, "%*c+ slice[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression? ':' expression? [':' expression?]")); @@ -14821,7 +14593,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_93_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_66_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -14859,15 +14631,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'(' (tuple | group | genexp)")); - void *_tmp_94_var; + void *_tmp_67_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7) // token='(' && - (_tmp_94_var = _tmp_94_rule(p)) // tuple | group | genexp + (_tmp_67_var = _tmp_67_rule(p)) // tuple | group | genexp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'(' (tuple | group | genexp)")); - _res = _tmp_94_var; + _res = _tmp_67_var; goto done; } p->mark = _mark; @@ -14880,15 +14652,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'[' (list | listcomp)")); - void *_tmp_95_var; + void *_tmp_68_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9) // token='[' && - (_tmp_95_var = _tmp_95_rule(p)) // list | listcomp + (_tmp_68_var = _tmp_68_rule(p)) // list | listcomp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'[' (list | listcomp)")); - _res = _tmp_95_var; + _res = _tmp_68_var; goto done; } p->mark = _mark; @@ -14901,15 +14673,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'{' (dict | set | dictcomp | setcomp)")); - void *_tmp_96_var; + void *_tmp_69_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25) // token='{' && - (_tmp_96_var = _tmp_96_rule(p)) // dict | set | dictcomp | setcomp + (_tmp_69_var = _tmp_69_rule(p)) // dict | set | dictcomp | setcomp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'{' (dict | set | dictcomp | setcomp)")); - _res = _tmp_96_var; + _res = _tmp_69_var; goto done; } p->mark = _mark; @@ -14980,7 +14752,7 @@ group_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_97_rule(p)) // yield_expr | named_expression + (a = _tmp_70_rule(p)) // yield_expr | named_expression && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -15181,9 +14953,9 @@ lambda_parameters_rule(Parser *p) if ( (a = lambda_slash_no_default_rule(p)) // lambda_slash_no_default && - (b = (asdl_arg_seq*)_loop0_98_rule(p)) // lambda_param_no_default* + (b = (asdl_arg_seq*)_loop0_71_rule(p)) // lambda_param_no_default* && - (c = _loop0_99_rule(p)) // lambda_param_with_default* + (c = _loop0_72_rule(p)) // lambda_param_with_default* && (d = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15213,7 +14985,7 @@ lambda_parameters_rule(Parser *p) if ( (a = lambda_slash_with_default_rule(p)) // lambda_slash_with_default && - (b = _loop0_100_rule(p)) // lambda_param_with_default* + (b = _loop0_72_rule(p)) // lambda_param_with_default* && (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15241,9 +15013,9 @@ lambda_parameters_rule(Parser *p) asdl_seq * b; void *c; if ( - (a = (asdl_arg_seq*)_loop1_101_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ && - (b = _loop0_102_rule(p)) // lambda_param_with_default* + (b = _loop0_72_rule(p)) // lambda_param_with_default* && (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15270,7 +15042,7 @@ lambda_parameters_rule(Parser *p) asdl_seq * a; void *b; if ( - (a = _loop1_103_rule(p)) // lambda_param_with_default+ + (a = _loop1_74_rule(p)) // lambda_param_with_default+ && (b = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15343,7 +15115,7 @@ lambda_slash_no_default_rule(Parser *p) Token * _literal_1; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_104_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15372,7 +15144,7 @@ lambda_slash_no_default_rule(Parser *p) Token * _literal; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_105_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15424,9 +15196,9 @@ lambda_slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_106_rule(p)) // lambda_param_no_default* + (a = _loop0_71_rule(p)) // lambda_param_no_default* && - (b = _loop1_107_rule(p)) // lambda_param_with_default+ + (b = _loop1_74_rule(p)) // lambda_param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15456,9 +15228,9 @@ lambda_slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_108_rule(p)) // lambda_param_no_default* + (a = _loop0_71_rule(p)) // lambda_param_no_default* && - (b = _loop1_109_rule(p)) // lambda_param_with_default+ + (b = _loop1_74_rule(p)) // lambda_param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15535,7 +15307,7 @@ lambda_star_etc_rule(Parser *p) && (a = lambda_param_no_default_rule(p)) // lambda_param_no_default && - (b = _loop0_110_rule(p)) // lambda_param_maybe_default* + (b = _loop0_75_rule(p)) // lambda_param_maybe_default* && (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) @@ -15568,7 +15340,7 @@ lambda_star_etc_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _loop1_111_rule(p)) // lambda_param_maybe_default+ + (b = _loop1_76_rule(p)) // lambda_param_maybe_default+ && (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) @@ -16208,7 +15980,7 @@ fstring_full_format_spec_rule(Parser *p) if ( (colon = _PyPegen_expect_token(p, 11)) // token=':' && - (spec = _loop0_112_rule(p)) // fstring_format_spec* + (spec = _loop0_77_rule(p)) // fstring_format_spec* ) { D(fprintf(stderr, "%*c+ fstring_full_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' fstring_format_spec*")); @@ -16326,7 +16098,7 @@ fstring_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' && - (b = _loop0_113_rule(p)) // fstring_middle* + (b = _loop0_78_rule(p)) // fstring_middle* && (c = _PyPegen_expect_token(p, FSTRING_END)) // token='FSTRING_END' ) @@ -16427,7 +16199,7 @@ strings_rule(Parser *p) D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_114_rule(p)) // ((fstring | string))+ + (a = (asdl_expr_seq*)_loop1_79_rule(p)) // ((fstring | string))+ ) { D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); @@ -16560,7 +16332,7 @@ tuple_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_115_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] + (a = _tmp_80_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -16775,7 +16547,7 @@ double_starred_kvpairs_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * a; if ( - (a = _gather_116_rule(p)) // ','.double_starred_kvpair+ + (a = _gather_82_rule(p)) // ','.double_starred_kvpair+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -16934,7 +16706,7 @@ for_if_clauses_rule(Parser *p) D(fprintf(stderr, "%*c> for_if_clauses[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); asdl_comprehension_seq* a; if ( - (a = (asdl_comprehension_seq*)_loop1_118_rule(p)) // for_if_clause+ + (a = (asdl_comprehension_seq*)_loop1_83_rule(p)) // for_if_clause+ ) { D(fprintf(stderr, "%*c+ for_if_clauses[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); @@ -16999,7 +16771,7 @@ for_if_clause_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_119_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_84_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); @@ -17042,7 +16814,7 @@ for_if_clause_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_120_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_84_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); @@ -17321,7 +17093,7 @@ genexp_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_121_rule(p)) // assignment_expression | expression !':=' + (a = _tmp_85_rule(p)) // assignment_expression | expression !':=' && (b = for_if_clauses_rule(p)) // for_if_clauses && @@ -17570,9 +17342,9 @@ args_rule(Parser *p) asdl_expr_seq* a; void *b; if ( - (a = (asdl_expr_seq*)_gather_122_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (a = (asdl_expr_seq*)_gather_87_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && - (b = _tmp_124_rule(p), !p->error_indicator) // [',' kwargs] + (b = _tmp_88_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -17662,11 +17434,11 @@ kwargs_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _gather_125_rule(p)) // ','.kwarg_or_starred+ + (a = _gather_90_rule(p)) // ','.kwarg_or_starred+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _gather_127_rule(p)) // ','.kwarg_or_double_starred+ + (b = _gather_92_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+")); @@ -17688,13 +17460,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - asdl_seq * _gather_129_var; + asdl_seq * _gather_90_var; if ( - (_gather_129_var = _gather_129_rule(p)) // ','.kwarg_or_starred+ + (_gather_90_var = _gather_90_rule(p)) // ','.kwarg_or_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - _res = _gather_129_var; + _res = _gather_90_var; goto done; } p->mark = _mark; @@ -17707,13 +17479,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - asdl_seq * _gather_131_var; + asdl_seq * _gather_92_var; if ( - (_gather_131_var = _gather_131_rule(p)) // ','.kwarg_or_double_starred+ + (_gather_92_var = _gather_92_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - _res = _gather_131_var; + _res = _gather_92_var; goto done; } p->mark = _mark; @@ -18124,7 +17896,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop0_133_rule(p)) // ((',' star_target))* + (b = _loop0_93_rule(p)) // ((',' star_target))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18180,7 +17952,7 @@ star_targets_list_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_134_rule(p)) // ','.star_target+ + (a = (asdl_expr_seq*)_gather_95_rule(p)) // ','.star_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18230,7 +18002,7 @@ star_targets_tuple_seq_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop1_136_rule(p)) // ((',' star_target))+ + (b = _loop1_96_rule(p)) // ((',' star_target))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18318,7 +18090,7 @@ star_target_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (a = _tmp_137_rule(p)) // !'*' star_target + (a = _tmp_97_rule(p)) // !'*' star_target ) { D(fprintf(stderr, "%*c+ star_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (!'*' star_target)")); @@ -19241,7 +19013,7 @@ del_targets_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_138_rule(p)) // ','.del_target+ + (a = (asdl_expr_seq*)_gather_99_rule(p)) // ','.del_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -19599,7 +19371,7 @@ type_expressions_rule(Parser *p) expr_ty b; expr_ty c; if ( - (a = _gather_140_rule(p)) // ','.expression+ + (a = _gather_101_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19638,7 +19410,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_142_rule(p)) // ','.expression+ + (a = _gather_101_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19671,7 +19443,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_144_rule(p)) // ','.expression+ + (a = _gather_101_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19791,7 +19563,7 @@ type_expressions_rule(Parser *p) D(fprintf(stderr, "%*c> type_expressions[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.expression+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_146_rule(p)) // ','.expression+ + (a = (asdl_expr_seq*)_gather_101_rule(p)) // ','.expression+ ) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+")); @@ -19842,7 +19614,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_102_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -19928,15 +19700,15 @@ invalid_arguments_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); - asdl_seq * _gather_150_var; - void *_tmp_149_var; + asdl_seq * _gather_105_var; + void *_tmp_103_var; Token * a; if ( - (_tmp_149_var = _tmp_149_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs + (_tmp_103_var = _tmp_103_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && (a = _PyPegen_expect_token(p, 12)) // token=',' && - (_gather_150_var = _gather_150_rule(p)) // ','.(starred_expression !'=')+ + (_gather_105_var = _gather_105_rule(p)) // ','.(starred_expression !'=')+ ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); @@ -19970,7 +19742,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_106_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -20030,13 +19802,13 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_153_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_107_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_154_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_108_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20174,7 +19946,7 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_155_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_109_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) @@ -20234,7 +20006,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_156_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_110_rule, p) && (a = expression_rule(p)) // expression && @@ -20575,7 +20347,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_157_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_111_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -20611,7 +20383,7 @@ invalid_expression_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_158_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_112_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -20732,7 +20504,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_159_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_113_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20758,7 +20530,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_160_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_114_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20766,7 +20538,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_161_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_113_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20846,7 +20618,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_162_var; + asdl_seq * _loop0_115_var; expr_ty a; expr_ty expression_var; if ( @@ -20854,7 +20626,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_162_var = _loop0_162_rule(p)) // star_named_expressions* + (_loop0_115_var = _loop0_115_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20911,10 +20683,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_116_var; expr_ty a; if ( - (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* + (_loop0_116_var = _loop0_116_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20941,10 +20713,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_164_var; + asdl_seq * _loop0_116_var; expr_ty a; if ( - (_loop0_164_var = _loop0_164_rule(p)) // ((star_targets '='))* + (_loop0_116_var = _loop0_116_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -21200,11 +20972,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_165_var; + void *_tmp_117_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_165_var = _tmp_165_rule(p)) // '[' | '(' | '{' + (_tmp_117_var = _tmp_117_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -21231,12 +21003,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_166_var; + void *_tmp_118_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' + (_tmp_118_var = _tmp_118_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21266,12 +21038,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_167_var; + void *_tmp_118_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' + (_tmp_118_var = _tmp_118_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21406,13 +21178,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_169_var; - void *_tmp_168_var; + asdl_seq * _loop0_32_var; + void *_tmp_119_var; Token * a; if ( - (_tmp_168_var = _tmp_168_rule(p)) // slash_no_default | slash_with_default + (_tmp_119_var = _tmp_119_rule(p)) // slash_no_default | slash_with_default && - (_loop0_169_var = _loop0_169_rule(p)) // param_maybe_default* + (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21436,7 +21208,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_170_var; + asdl_seq * _loop0_28_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21444,7 +21216,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* + (_loop0_28_var = _loop0_28_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -21470,18 +21242,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_171_var; - asdl_seq * _loop1_172_var; + asdl_seq * _loop0_28_var; + asdl_seq * _loop1_30_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* + (_loop0_28_var = _loop0_28_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_172_var = _loop1_172_rule(p)) // param_no_default+ + (_loop1_30_var = _loop1_30_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21508,22 +21280,22 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_174_var; - asdl_seq * _loop0_176_var; + asdl_seq * _loop0_32_var; + asdl_seq * _loop0_32_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_175_var; + void *_tmp_120_var; Token * a; if ( - (_opt_var = _tmp_173_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_119_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_174_var = _loop0_174_rule(p)) // param_maybe_default* + (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_175_var = _tmp_175_rule(p)) // ',' | param_no_default + (_tmp_120_var = _tmp_120_rule(p)) // ',' | param_no_default && - (_loop0_176_var = _loop0_176_rule(p)) // param_maybe_default* + (_loop0_32_var_1 = _loop0_32_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21548,10 +21320,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_177_var; + asdl_seq * _loop1_33_var; Token * a; if ( - (_loop1_177_var = _loop1_177_rule(p)) // param_maybe_default+ + (_loop1_33_var = _loop1_33_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21600,7 +21372,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_178_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_121_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -21645,12 +21417,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_179_var; + void *_tmp_122_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_179_var = _tmp_179_rule(p)) // ')' | ',' (')' | '**') + (_tmp_122_var = _tmp_122_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -21733,20 +21505,20 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_181_var; - void *_tmp_180_var; - void *_tmp_182_var; + asdl_seq * _loop0_32_var; + void *_tmp_123_var; + void *_tmp_123_var_1; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_180_var = _tmp_180_rule(p)) // param_no_default | ',' + (_tmp_123_var = _tmp_123_rule(p)) // param_no_default | ',' && - (_loop0_181_var = _loop0_181_rule(p)) // param_maybe_default* + (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_182_var = _tmp_182_rule(p)) // param_no_default | ',' + (_tmp_123_var_1 = _tmp_123_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -21861,7 +21633,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_183_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_124_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -21926,13 +21698,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_184_var; + asdl_seq * _loop1_31_var; if ( - (_loop1_184_var = _loop1_184_rule(p)) // param_with_default+ + (_loop1_31_var = _loop1_31_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_184_var; + _res = _loop1_31_var; goto done; } p->mark = _mark; @@ -21997,13 +21769,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_186_var; - void *_tmp_185_var; + asdl_seq * _loop0_75_var; + void *_tmp_125_var; Token * a; if ( - (_tmp_185_var = _tmp_185_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_125_var = _tmp_125_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_maybe_default* + (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22027,7 +21799,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_187_var; + asdl_seq * _loop0_71_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -22035,7 +21807,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* + (_loop0_71_var = _loop0_71_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -22061,18 +21833,18 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_189_var; - asdl_seq * _loop0_188_var; + asdl_seq * _gather_127_var; + asdl_seq * _loop0_71_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* + (_loop0_71_var = _loop0_71_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_189_var = _gather_189_rule(p)) // ','.lambda_param+ + (_gather_127_var = _gather_127_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22099,22 +21871,22 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_192_var; - asdl_seq * _loop0_194_var; + asdl_seq * _loop0_75_var; + asdl_seq * _loop0_75_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_193_var; + void *_tmp_128_var; Token * a; if ( - (_opt_var = _tmp_191_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_125_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_192_var = _loop0_192_rule(p)) // lambda_param_maybe_default* + (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_193_var = _tmp_193_rule(p)) // ',' | lambda_param_no_default + (_tmp_128_var = _tmp_128_rule(p)) // ',' | lambda_param_no_default && - (_loop0_194_var = _loop0_194_rule(p)) // lambda_param_maybe_default* + (_loop0_75_var_1 = _loop0_75_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22139,10 +21911,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_195_var; + asdl_seq * _loop1_76_var; Token * a; if ( - (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_maybe_default+ + (_loop1_76_var = _loop1_76_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -22213,13 +21985,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_196_var; + asdl_seq * _loop1_74_var; if ( - (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_with_default+ + (_loop1_74_var = _loop1_74_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_196_var; + _res = _loop1_74_var; goto done; } p->mark = _mark; @@ -22255,11 +22027,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_197_var; + void *_tmp_129_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_197_var = _tmp_197_rule(p)) // ':' | ',' (':' | '**') + (_tmp_129_var = _tmp_129_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -22312,20 +22084,20 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_199_var; - void *_tmp_198_var; - void *_tmp_200_var; + asdl_seq * _loop0_75_var; + void *_tmp_130_var; + void *_tmp_130_var_1; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_198_var = _tmp_198_rule(p)) // lambda_param_no_default | ',' + (_tmp_130_var = _tmp_130_rule(p)) // lambda_param_no_default | ',' && - (_loop0_199_var = _loop0_199_rule(p)) // lambda_param_maybe_default* + (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_200_var = _tmp_200_rule(p)) // lambda_param_no_default | ',' + (_tmp_130_var_1 = _tmp_130_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -22443,7 +22215,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_201_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_124_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -22549,7 +22321,7 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_202_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22593,13 +22365,13 @@ invalid_for_if_clause_rule(Parser *p) Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_203_var; + void *_tmp_131_var; if ( (_opt_var = _PyPegen_expect_token(p, 682), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 678)) // token='for' && - (_tmp_203_var = _tmp_203_rule(p)) // bitwise_or ((',' bitwise_or))* ','? + (_tmp_131_var = _tmp_131_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 679) // token='in' ) @@ -22774,14 +22546,14 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_204_var; + asdl_seq * _gather_133_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( (a = _PyPegen_expect_token(p, 622)) // token='import' && - (_gather_204_var = _gather_204_rule(p)) // ','.dotted_name+ + (_gather_133_var = _gather_133_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 621)) // token='from' && @@ -22928,7 +22700,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_206_var; + asdl_seq * _gather_135_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22938,7 +22710,7 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && - (_gather_206_var = _gather_206_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_135_var = _gather_135_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22962,7 +22734,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_208_var; + asdl_seq * _gather_137_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -22978,7 +22750,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_208_var = _gather_208_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_137_var = _gather_137_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -23027,7 +22799,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_210_var; + asdl_seq * _gather_135_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -23038,7 +22810,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 633)) // token='with' && - (_gather_210_var = _gather_210_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_135_var = _gather_135_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23066,7 +22838,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_212_var; + asdl_seq * _gather_137_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -23083,7 +22855,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_212_var = _gather_212_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_137_var = _gather_137_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -23180,7 +22952,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_214_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -23205,8 +22977,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_215_var; - asdl_seq * _loop1_216_var; + asdl_seq * _loop0_139_var; + asdl_seq * _loop1_37_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -23217,9 +22989,9 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_215_var = _loop0_215_rule(p)) // block* + (_loop0_139_var = _loop0_139_rule(p)) // block* && - (_loop1_216_var = _loop1_216_rule(p)) // except_block+ + (_loop1_37_var = _loop1_37_rule(p)) // except_block+ && (a = _PyPegen_expect_token(p, 663)) // token='except' && @@ -23227,7 +22999,7 @@ invalid_try_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_217_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23254,8 +23026,8 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_218_var; - asdl_seq * _loop1_219_var; + asdl_seq * _loop0_139_var; + asdl_seq * _loop1_38_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; @@ -23264,13 +23036,13 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_218_var = _loop0_218_rule(p)) // block* + (_loop0_139_var = _loop0_139_rule(p)) // block* && - (_loop1_219_var = _loop1_219_rule(p)) // except_star_block+ + (_loop1_38_var = _loop1_38_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 663)) // token='except' && - (_opt_var = _tmp_220_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_140_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23333,7 +23105,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_221_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23367,7 +23139,7 @@ invalid_except_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_222_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23493,7 +23265,7 @@ invalid_except_star_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_223_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -23530,7 +23302,7 @@ invalid_except_star_stmt_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_224_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23555,14 +23327,14 @@ invalid_except_star_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_225_var; + void *_tmp_141_var; Token * a; if ( (a = _PyPegen_expect_token(p, 663)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_225_var = _tmp_225_rule(p)) // NEWLINE | ':' + (_tmp_141_var = _tmp_141_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -23703,7 +23475,7 @@ invalid_except_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_226_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23797,7 +23569,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_227_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24159,7 +23931,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_228_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_142_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -24652,7 +24424,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_229_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24711,7 +24483,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_230_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -24769,7 +24541,7 @@ invalid_class_def_raw_rule(Parser *p) && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_231_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24808,7 +24580,7 @@ invalid_class_def_raw_rule(Parser *p) && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_232_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24858,11 +24630,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_233_var; + asdl_seq * _gather_82_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_233_var = _gather_233_rule(p)) // ','.double_starred_kvpair+ + (_gather_82_var = _gather_82_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24870,7 +24642,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_233_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_82_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24923,7 +24695,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_235_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_143_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -25033,7 +24805,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_236_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_143_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -25321,7 +25093,7 @@ invalid_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_237_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_144_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -25353,7 +25125,7 @@ invalid_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_238_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_145_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -25417,9 +25189,9 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_239_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_146_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_240_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_147_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -25443,7 +25215,7 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_242_var; + asdl_seq * _loop0_77_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -25456,11 +25228,11 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_146_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_242_var = _loop0_242_rule(p)) // fstring_format_spec* + (_loop0_77_var = _loop0_77_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25497,7 +25269,7 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_243_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_146_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25544,7 +25316,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_244_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_147_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25611,14 +25383,14 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_245_var; + void *_tmp_148_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_245_var = _tmp_245_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_148_var = _tmp_148_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && (a = _PyPegen_expect_token(p, 687)) // token='not' && @@ -25663,11 +25435,11 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_246_var; + void *_tmp_149_var; Token * a; expr_ty b; if ( - (_tmp_246_var = _tmp_246_rule(p)) // '+' | '-' | '~' + (_tmp_149_var = _tmp_149_rule(p)) // '+' | '-' | '~' && (a = _PyPegen_expect_token(p, 687)) // token='not' && @@ -25806,76 +25578,9 @@ _loop0_1_rule(Parser *p) return _seq; } -// _loop0_2: NEWLINE -static asdl_seq * -_loop0_2_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // NEWLINE - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_2[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); - Token * newline_var; - while ( - (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' - ) - { - _res = newline_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_2[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_3: statement +// _loop1_2: statement static asdl_seq * -_loop1_3_rule(Parser *p) +_loop1_2_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -25900,7 +25605,7 @@ _loop1_3_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_3[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "statement")); + D(fprintf(stderr, "%*c> _loop1_2[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "statement")); asdl_stmt_seq* statement_var; while ( (statement_var = statement_rule(p)) // statement @@ -25923,7 +25628,7 @@ _loop1_3_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_3[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_2[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "statement")); } if (_n == 0 || p->error_indicator) { @@ -25945,9 +25650,9 @@ _loop1_3_rule(Parser *p) return _seq; } -// _loop0_5: ';' simple_stmt +// _loop0_3: ';' simple_stmt static asdl_seq * -_loop0_5_rule(Parser *p) +_loop0_3_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -25972,7 +25677,7 @@ _loop0_5_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';' simple_stmt")); + D(fprintf(stderr, "%*c> _loop0_3[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';' simple_stmt")); Token * _literal; stmt_ty elem; while ( @@ -26004,7 +25709,7 @@ _loop0_5_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_5[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_3[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "';' simple_stmt")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -26021,7 +25726,7 @@ _loop0_5_rule(Parser *p) return _seq; } -// _gather_4: simple_stmt _loop0_5 +// _gather_4: simple_stmt _loop0_3 static asdl_seq * _gather_4_rule(Parser *p) { @@ -26034,27 +25739,27 @@ _gather_4_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // simple_stmt _loop0_5 + { // simple_stmt _loop0_3 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_4[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "simple_stmt _loop0_5")); + D(fprintf(stderr, "%*c> _gather_4[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "simple_stmt _loop0_3")); stmt_ty elem; asdl_seq * seq; if ( (elem = simple_stmt_rule(p)) // simple_stmt && - (seq = _loop0_5_rule(p)) // _loop0_5 + (seq = _loop0_3_rule(p)) // _loop0_3 ) { - D(fprintf(stderr, "%*c+ _gather_4[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "simple_stmt _loop0_5")); + D(fprintf(stderr, "%*c+ _gather_4[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "simple_stmt _loop0_3")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _gather_4[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "simple_stmt _loop0_5")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "simple_stmt _loop0_3")); } _res = NULL; done: @@ -26062,9 +25767,9 @@ _gather_4_rule(Parser *p) return _res; } -// _tmp_6: 'import' | 'from' +// _tmp_5: 'import' | 'from' static void * -_tmp_6_rule(Parser *p) +_tmp_5_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26080,18 +25785,18 @@ _tmp_6_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); + D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 622)) // token='import' ) { - D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); + D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_6[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_5[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'import'")); } { // 'from' @@ -26099,18 +25804,18 @@ _tmp_6_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); + D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 621)) // token='from' ) { - D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); + D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_6[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_5[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from'")); } _res = NULL; @@ -26119,9 +25824,9 @@ _tmp_6_rule(Parser *p) return _res; } -// _tmp_7: 'def' | '@' | 'async' +// _tmp_6: 'def' | '@' | 'async' static void * -_tmp_7_rule(Parser *p) +_tmp_6_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26137,18 +25842,18 @@ _tmp_7_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); + D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 683)) // token='def' ) { - D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); + D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_7[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_6[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def'")); } { // '@' @@ -26156,18 +25861,18 @@ _tmp_7_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_7[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_6[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } { // 'async' @@ -26175,18 +25880,18 @@ _tmp_7_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); + D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 682)) // token='async' ) { - D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); + D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_7[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_6[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'")); } _res = NULL; @@ -26195,9 +25900,9 @@ _tmp_7_rule(Parser *p) return _res; } -// _tmp_8: 'class' | '@' +// _tmp_7: 'class' | '@' static void * -_tmp_8_rule(Parser *p) +_tmp_7_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26213,18 +25918,18 @@ _tmp_8_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); + D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 685)) // token='class' ) { - D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); + D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_8[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_7[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'class'")); } { // '@' @@ -26232,18 +25937,18 @@ _tmp_8_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_8[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_7[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -26252,9 +25957,9 @@ _tmp_8_rule(Parser *p) return _res; } -// _tmp_9: 'with' | 'async' +// _tmp_8: 'with' | 'async' static void * -_tmp_9_rule(Parser *p) +_tmp_8_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26270,18 +25975,18 @@ _tmp_9_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); + D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 633)) // token='with' ) { - D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); + D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_9[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_8[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'with'")); } { // 'async' @@ -26289,18 +25994,18 @@ _tmp_9_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); + D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 682)) // token='async' ) { - D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); + D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_9[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_8[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'")); } _res = NULL; @@ -26309,9 +26014,9 @@ _tmp_9_rule(Parser *p) return _res; } -// _tmp_10: 'for' | 'async' +// _tmp_9: 'for' | 'async' static void * -_tmp_10_rule(Parser *p) +_tmp_9_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26327,18 +26032,18 @@ _tmp_10_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); + D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 678)) // token='for' ) { - D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); + D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_10[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_9[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'for'")); } { // 'async' @@ -26346,18 +26051,18 @@ _tmp_10_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); + D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 682)) // token='async' ) { - D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); + D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_10[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_9[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'")); } _res = NULL; @@ -26366,9 +26071,9 @@ _tmp_10_rule(Parser *p) return _res; } -// _tmp_11: '=' annotated_rhs +// _tmp_10: '=' annotated_rhs static void * -_tmp_11_rule(Parser *p) +_tmp_10_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26384,7 +26089,7 @@ _tmp_11_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_11[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); + D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); Token * _literal; expr_ty d; if ( @@ -26393,7 +26098,7 @@ _tmp_11_rule(Parser *p) (d = annotated_rhs_rule(p)) // annotated_rhs ) { - D(fprintf(stderr, "%*c+ _tmp_11[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); + D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); _res = d; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -26403,7 +26108,7 @@ _tmp_11_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_11[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_10[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' annotated_rhs")); } _res = NULL; @@ -26412,9 +26117,9 @@ _tmp_11_rule(Parser *p) return _res; } -// _tmp_12: '(' single_target ')' | single_subscript_attribute_target +// _tmp_11: '(' single_target ')' | single_subscript_attribute_target static void * -_tmp_12_rule(Parser *p) +_tmp_11_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26430,7 +26135,7 @@ _tmp_12_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' single_target ')'")); + D(fprintf(stderr, "%*c> _tmp_11[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' single_target ')'")); Token * _literal; Token * _literal_1; expr_ty b; @@ -26442,7 +26147,7 @@ _tmp_12_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_12[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' single_target ')'")); + D(fprintf(stderr, "%*c+ _tmp_11[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' single_target ')'")); _res = b; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -26452,7 +26157,7 @@ _tmp_12_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_12[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_11[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' single_target ')'")); } { // single_subscript_attribute_target @@ -26460,18 +26165,18 @@ _tmp_12_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_subscript_attribute_target")); + D(fprintf(stderr, "%*c> _tmp_11[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_subscript_attribute_target")); expr_ty single_subscript_attribute_target_var; if ( (single_subscript_attribute_target_var = single_subscript_attribute_target_rule(p)) // single_subscript_attribute_target ) { - D(fprintf(stderr, "%*c+ _tmp_12[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_subscript_attribute_target")); + D(fprintf(stderr, "%*c+ _tmp_11[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_subscript_attribute_target")); _res = single_subscript_attribute_target_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_12[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_11[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "single_subscript_attribute_target")); } _res = NULL; @@ -26480,55 +26185,9 @@ _tmp_12_rule(Parser *p) return _res; } -// _tmp_13: '=' annotated_rhs -static void * -_tmp_13_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '=' annotated_rhs - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_13[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); - Token * _literal; - expr_ty d; - if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - && - (d = annotated_rhs_rule(p)) // annotated_rhs - ) - { - D(fprintf(stderr, "%*c+ _tmp_13[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); - _res = d; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_13[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' annotated_rhs")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_14: (star_targets '=') +// _loop1_12: (star_targets '=') static asdl_seq * -_loop1_14_rule(Parser *p) +_loop1_12_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26553,13 +26212,13 @@ _loop1_14_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_247_var; + D(fprintf(stderr, "%*c> _loop1_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_150_var; while ( - (_tmp_247_var = _tmp_247_rule(p)) // star_targets '=' + (_tmp_150_var = _tmp_150_rule(p)) // star_targets '=' ) { - _res = _tmp_247_var; + _res = _tmp_150_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26576,7 +26235,7 @@ _loop1_14_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_14[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_12[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } if (_n == 0 || p->error_indicator) { @@ -26598,123 +26257,9 @@ _loop1_14_rule(Parser *p) return _seq; } -// _tmp_15: yield_expr | star_expressions -static void * -_tmp_15_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_15[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; - if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions - ) - { - D(fprintf(stderr, "%*c+ _tmp_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_15[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_16: yield_expr | star_expressions -static void * -_tmp_16_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // star_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_expressions")); - expr_ty star_expressions_var; - if ( - (star_expressions_var = star_expressions_rule(p)) // star_expressions - ) - { - D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions")); - _res = star_expressions_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_expressions")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_17: 'from' expression +// _tmp_13: 'from' expression static void * -_tmp_17_rule(Parser *p) +_tmp_13_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26730,7 +26275,7 @@ _tmp_17_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_17[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from' expression")); + D(fprintf(stderr, "%*c> _tmp_13[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from' expression")); Token * _keyword; expr_ty z; if ( @@ -26739,7 +26284,7 @@ _tmp_17_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_17[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' expression")); + D(fprintf(stderr, "%*c+ _tmp_13[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -26749,7 +26294,7 @@ _tmp_17_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_17[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_13[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from' expression")); } _res = NULL; @@ -26758,126 +26303,9 @@ _tmp_17_rule(Parser *p) return _res; } -// _loop0_19: ',' NAME -static asdl_seq * -_loop0_19_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' NAME - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _PyPegen_name_token(p)) // NAME - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_19[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' NAME")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_18: NAME _loop0_19 -static asdl_seq * -_gather_18_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // NAME _loop0_19 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_19")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = _PyPegen_name_token(p)) // NAME - && - (seq = _loop0_19_rule(p)) // _loop0_19 - ) - { - D(fprintf(stderr, "%*c+ _gather_18[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_19")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_18[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_19")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_21: ',' NAME +// _loop0_14: ',' NAME static asdl_seq * -_loop0_21_rule(Parser *p) +_loop0_14_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26902,7 +26330,7 @@ _loop0_21_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_21[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); + D(fprintf(stderr, "%*c> _loop0_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); Token * _literal; expr_ty elem; while ( @@ -26934,7 +26362,7 @@ _loop0_21_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_21[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_14[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' NAME")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -26951,9 +26379,9 @@ _loop0_21_rule(Parser *p) return _seq; } -// _gather_20: NAME _loop0_21 +// _gather_15: NAME _loop0_14 static asdl_seq * -_gather_20_rule(Parser *p) +_gather_15_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -26964,27 +26392,27 @@ _gather_20_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // NAME _loop0_21 + { // NAME _loop0_14 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_20[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_21")); + D(fprintf(stderr, "%*c> _gather_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_14")); expr_ty elem; asdl_seq * seq; if ( (elem = _PyPegen_name_token(p)) // NAME && - (seq = _loop0_21_rule(p)) // _loop0_21 + (seq = _loop0_14_rule(p)) // _loop0_14 ) { - D(fprintf(stderr, "%*c+ _gather_20[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_21")); + D(fprintf(stderr, "%*c+ _gather_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_14")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_20[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_21")); + D(fprintf(stderr, "%*c%s _gather_15[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_14")); } _res = NULL; done: @@ -26992,9 +26420,9 @@ _gather_20_rule(Parser *p) return _res; } -// _tmp_22: ';' | NEWLINE +// _tmp_16: ';' | NEWLINE static void * -_tmp_22_rule(Parser *p) +_tmp_16_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27010,18 +26438,18 @@ _tmp_22_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); + D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 13)) // token=';' ) { - D(fprintf(stderr, "%*c+ _tmp_22[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); + D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_22[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "';'")); } { // NEWLINE @@ -27029,18 +26457,18 @@ _tmp_22_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_22[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_22[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } _res = NULL; @@ -27049,9 +26477,9 @@ _tmp_22_rule(Parser *p) return _res; } -// _tmp_23: ',' expression +// _tmp_17: ',' expression static void * -_tmp_23_rule(Parser *p) +_tmp_17_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27067,7 +26495,7 @@ _tmp_23_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_23[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_17[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty z; if ( @@ -27076,7 +26504,7 @@ _tmp_23_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_23[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_17[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -27086,7 +26514,7 @@ _tmp_23_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_23[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_17[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -27095,9 +26523,9 @@ _tmp_23_rule(Parser *p) return _res; } -// _loop0_24: ('.' | '...') +// _loop0_18: ('.' | '...') static asdl_seq * -_loop0_24_rule(Parser *p) +_loop0_18_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27122,13 +26550,13 @@ _loop0_24_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_248_var; + D(fprintf(stderr, "%*c> _loop0_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_151_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // '.' | '...' + (_tmp_151_var = _tmp_151_rule(p)) // '.' | '...' ) { - _res = _tmp_248_var; + _res = _tmp_151_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27145,7 +26573,7 @@ _loop0_24_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_24[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_18[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -27162,9 +26590,9 @@ _loop0_24_rule(Parser *p) return _seq; } -// _loop1_25: ('.' | '...') +// _loop1_19: ('.' | '...') static asdl_seq * -_loop1_25_rule(Parser *p) +_loop1_19_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27189,13 +26617,13 @@ _loop1_25_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_249_var; + D(fprintf(stderr, "%*c> _loop1_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_151_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // '.' | '...' + (_tmp_151_var = _tmp_151_rule(p)) // '.' | '...' ) { - _res = _tmp_249_var; + _res = _tmp_151_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27212,7 +26640,7 @@ _loop1_25_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_25[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_19[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } if (_n == 0 || p->error_indicator) { @@ -27234,9 +26662,9 @@ _loop1_25_rule(Parser *p) return _seq; } -// _loop0_27: ',' import_from_as_name +// _loop0_20: ',' import_from_as_name static asdl_seq * -_loop0_27_rule(Parser *p) +_loop0_20_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27261,7 +26689,7 @@ _loop0_27_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_27[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' import_from_as_name")); + D(fprintf(stderr, "%*c> _loop0_20[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' import_from_as_name")); Token * _literal; alias_ty elem; while ( @@ -27293,7 +26721,7 @@ _loop0_27_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_27[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_20[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' import_from_as_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -27310,9 +26738,9 @@ _loop0_27_rule(Parser *p) return _seq; } -// _gather_26: import_from_as_name _loop0_27 +// _gather_21: import_from_as_name _loop0_20 static asdl_seq * -_gather_26_rule(Parser *p) +_gather_21_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27323,27 +26751,27 @@ _gather_26_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // import_from_as_name _loop0_27 + { // import_from_as_name _loop0_20 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_27")); + D(fprintf(stderr, "%*c> _gather_21[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_20")); alias_ty elem; asdl_seq * seq; if ( (elem = import_from_as_name_rule(p)) // import_from_as_name && - (seq = _loop0_27_rule(p)) // _loop0_27 + (seq = _loop0_20_rule(p)) // _loop0_20 ) { - D(fprintf(stderr, "%*c+ _gather_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_27")); + D(fprintf(stderr, "%*c+ _gather_21[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_20")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_26[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_name _loop0_27")); + D(fprintf(stderr, "%*c%s _gather_21[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_name _loop0_20")); } _res = NULL; done: @@ -27351,9 +26779,9 @@ _gather_26_rule(Parser *p) return _res; } -// _tmp_28: 'as' NAME +// _tmp_22: 'as' NAME static void * -_tmp_28_rule(Parser *p) +_tmp_22_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27369,7 +26797,7 @@ _tmp_28_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_28[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty z; if ( @@ -27378,7 +26806,7 @@ _tmp_28_rule(Parser *p) (z = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_28[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_22[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -27388,7 +26816,7 @@ _tmp_28_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_28[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_22[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -27397,9 +26825,9 @@ _tmp_28_rule(Parser *p) return _res; } -// _loop0_30: ',' dotted_as_name +// _loop0_23: ',' dotted_as_name static asdl_seq * -_loop0_30_rule(Parser *p) +_loop0_23_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27424,7 +26852,7 @@ _loop0_30_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_30[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_as_name")); + D(fprintf(stderr, "%*c> _loop0_23[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_as_name")); Token * _literal; alias_ty elem; while ( @@ -27456,7 +26884,7 @@ _loop0_30_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_30[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_23[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_as_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -27473,9 +26901,9 @@ _loop0_30_rule(Parser *p) return _seq; } -// _gather_29: dotted_as_name _loop0_30 +// _gather_24: dotted_as_name _loop0_23 static asdl_seq * -_gather_29_rule(Parser *p) +_gather_24_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27486,27 +26914,27 @@ _gather_29_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_as_name _loop0_30 + { // dotted_as_name _loop0_23 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_29[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_30")); + D(fprintf(stderr, "%*c> _gather_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_23")); alias_ty elem; asdl_seq * seq; if ( (elem = dotted_as_name_rule(p)) // dotted_as_name && - (seq = _loop0_30_rule(p)) // _loop0_30 + (seq = _loop0_23_rule(p)) // _loop0_23 ) { - D(fprintf(stderr, "%*c+ _gather_29[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_30")); + D(fprintf(stderr, "%*c+ _gather_24[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_23")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_29[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_as_name _loop0_30")); + D(fprintf(stderr, "%*c%s _gather_24[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_as_name _loop0_23")); } _res = NULL; done: @@ -27514,55 +26942,9 @@ _gather_29_rule(Parser *p) return _res; } -// _tmp_31: 'as' NAME -static void * -_tmp_31_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'as' NAME - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_31[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty z; - if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (z = _PyPegen_name_token(p)) // NAME - ) - { - D(fprintf(stderr, "%*c+ _tmp_31[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_31[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_32: ('@' named_expression NEWLINE) +// _loop1_25: ('@' named_expression NEWLINE) static asdl_seq * -_loop1_32_rule(Parser *p) +_loop1_25_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27587,13 +26969,13 @@ _loop1_32_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_250_var; + D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); + void *_tmp_152_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // '@' named_expression NEWLINE + (_tmp_152_var = _tmp_152_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_250_var; + _res = _tmp_152_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27610,7 +26992,7 @@ _loop1_32_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_32[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_25[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('@' named_expression NEWLINE)")); } if (_n == 0 || p->error_indicator) { @@ -27632,9 +27014,9 @@ _loop1_32_rule(Parser *p) return _seq; } -// _tmp_33: '(' arguments? ')' +// _tmp_26: '(' arguments? ')' static void * -_tmp_33_rule(Parser *p) +_tmp_26_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27650,7 +27032,7 @@ _tmp_33_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *z; @@ -27662,7 +27044,7 @@ _tmp_33_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_33[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -27672,7 +27054,7 @@ _tmp_33_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_33[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_26[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -27681,9 +27063,9 @@ _tmp_33_rule(Parser *p) return _res; } -// _tmp_34: '->' expression +// _tmp_27: '->' expression static void * -_tmp_34_rule(Parser *p) +_tmp_27_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27699,7 +27081,7 @@ _tmp_34_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_34[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_27[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty z; if ( @@ -27708,7 +27090,7 @@ _tmp_34_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_34[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_27[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -27718,7 +27100,7 @@ _tmp_34_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_34[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_27[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -27727,55 +27109,9 @@ _tmp_34_rule(Parser *p) return _res; } -// _tmp_35: '->' expression -static void * -_tmp_35_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '->' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); - Token * _literal; - expr_ty z; - if ( - (_literal = _PyPegen_expect_token(p, 51)) // token='->' - && - (z = expression_rule(p)) // expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_36: param_no_default +// _loop0_28: param_no_default static asdl_seq * -_loop0_36_rule(Parser *p) +_loop0_28_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27800,7 +27136,7 @@ _loop0_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_28[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -27823,7 +27159,7 @@ _loop0_36_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_28[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -27840,76 +27176,9 @@ _loop0_36_rule(Parser *p) return _seq; } -// _loop0_37: param_with_default -static asdl_seq * -_loop0_37_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_37[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); - NameDefaultPair* param_with_default_var; - while ( - (param_with_default_var = param_with_default_rule(p)) // param_with_default - ) - { - _res = param_with_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_37[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_38: param_with_default +// _loop0_29: param_with_default static asdl_seq * -_loop0_38_rule(Parser *p) +_loop0_29_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -27934,7 +27203,7 @@ _loop0_38_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_38[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop0_29[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -27957,7 +27226,7 @@ _loop0_38_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_38[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_29[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -27974,9 +27243,9 @@ _loop0_38_rule(Parser *p) return _seq; } -// _loop1_39: param_no_default +// _loop1_30: param_no_default static asdl_seq * -_loop1_39_rule(Parser *p) +_loop1_30_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28001,7 +27270,7 @@ _loop1_39_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_39[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_30[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -28024,7 +27293,7 @@ _loop1_39_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_39[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_30[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -28046,9 +27315,9 @@ _loop1_39_rule(Parser *p) return _seq; } -// _loop0_40: param_with_default +// _loop1_31: param_with_default static asdl_seq * -_loop0_40_rule(Parser *p) +_loop1_31_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28073,7 +27342,7 @@ _loop0_40_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_40[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_31[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -28096,9 +27365,14 @@ _loop0_40_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_40[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_31[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { PyMem_Free(_children); @@ -28113,9 +27387,9 @@ _loop0_40_rule(Parser *p) return _seq; } -// _loop1_41: param_with_default +// _loop0_32: param_maybe_default static asdl_seq * -_loop1_41_rule(Parser *p) +_loop0_32_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28135,18 +27409,18 @@ _loop1_41_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_with_default + { // param_maybe_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); - NameDefaultPair* param_with_default_var; + D(fprintf(stderr, "%*c> _loop0_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + NameDefaultPair* param_maybe_default_var; while ( - (param_with_default_var = param_with_default_rule(p)) // param_with_default + (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default ) { - _res = param_with_default_var; + _res = param_maybe_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28163,13 +27437,8 @@ _loop1_41_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_41[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_32[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -28185,9 +27454,9 @@ _loop1_41_rule(Parser *p) return _seq; } -// _loop1_42: param_no_default +// _loop1_33: param_maybe_default static asdl_seq * -_loop1_42_rule(Parser *p) +_loop1_33_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28207,18 +27476,18 @@ _loop1_42_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_no_default + { // param_maybe_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; + D(fprintf(stderr, "%*c> _loop1_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + NameDefaultPair* param_maybe_default_var; while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default + (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default ) { - _res = param_no_default_var; + _res = param_maybe_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28235,8 +27504,8 @@ _loop1_42_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_42[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c%s _loop1_33[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -28257,9 +27526,9 @@ _loop1_42_rule(Parser *p) return _seq; } -// _loop1_43: param_no_default +// _loop0_34: ',' with_item static asdl_seq * -_loop1_43_rule(Parser *p) +_loop0_34_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28279,18 +27548,27 @@ _loop1_43_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_no_default + { // ',' with_item if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; + D(fprintf(stderr, "%*c> _loop0_34[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + Token * _literal; + withitem_ty elem; while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = with_item_rule(p)) // with_item ) { - _res = param_no_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28307,13 +27585,8 @@ _loop1_43_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_43[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_34[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -28329,9 +27602,9 @@ _loop1_43_rule(Parser *p) return _seq; } -// _loop0_44: param_no_default +// _gather_35: with_item _loop0_34 static asdl_seq * -_loop0_44_rule(Parser *p) +_gather_35_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28340,65 +27613,39 @@ _loop0_44_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_no_default + { // with_item _loop0_34 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default + D(fprintf(stderr, "%*c> _gather_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_34")); + withitem_ty elem; + asdl_seq * seq; + if ( + (elem = with_item_rule(p)) // with_item + && + (seq = _loop0_34_rule(p)) // _loop0_34 ) { - _res = param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _gather_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_34")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_44[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _gather_35[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_34")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop1_45: param_with_default -static asdl_seq * -_loop1_45_rule(Parser *p) +// _tmp_36: ',' | ')' | ':' +static void * +_tmp_36_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28407,70 +27654,74 @@ _loop1_45_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_with_default + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_45[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); - NameDefaultPair* param_with_default_var; - while ( - (param_with_default_var = param_with_default_rule(p)) // param_with_default + D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - _res = param_with_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + _res = _literal; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_45[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + { // ')' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 8)) // token=')' + ) + { + D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + { // ':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + ) + { + D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop0_46: param_no_default +// _loop1_37: except_block static asdl_seq * -_loop0_46_rule(Parser *p) +_loop1_37_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28490,18 +27741,18 @@ _loop0_46_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_no_default + { // except_block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_46[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; + D(fprintf(stderr, "%*c> _loop1_37[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + excepthandler_ty except_block_var; while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default + (except_block_var = except_block_rule(p)) // except_block ) { - _res = param_no_default_var; + _res = except_block_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28518,8 +27769,13 @@ _loop0_46_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_46[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c%s _loop1_37[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -28535,9 +27791,9 @@ _loop0_46_rule(Parser *p) return _seq; } -// _loop1_47: param_with_default +// _loop1_38: except_star_block static asdl_seq * -_loop1_47_rule(Parser *p) +_loop1_38_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28557,18 +27813,18 @@ _loop1_47_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_with_default + { // except_star_block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_47[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); - NameDefaultPair* param_with_default_var; + D(fprintf(stderr, "%*c> _loop1_38[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + excepthandler_ty except_star_block_var; while ( - (param_with_default_var = param_with_default_rule(p)) // param_with_default + (except_star_block_var = except_star_block_rule(p)) // except_star_block ) { - _res = param_with_default_var; + _res = except_star_block_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28585,8 +27841,8 @@ _loop1_47_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_47[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c%s _loop1_38[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -28607,9 +27863,9 @@ _loop1_47_rule(Parser *p) return _seq; } -// _loop0_48: param_maybe_default +// _loop1_39: case_block static asdl_seq * -_loop0_48_rule(Parser *p) +_loop1_39_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28629,18 +27885,18 @@ _loop0_48_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_maybe_default + { // case_block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; + D(fprintf(stderr, "%*c> _loop1_39[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "case_block")); + match_case_ty case_block_var; while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default + (case_block_var = case_block_rule(p)) // case_block ) { - _res = param_maybe_default_var; + _res = case_block_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28657,8 +27913,13 @@ _loop0_48_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_48[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c%s _loop1_39[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "case_block")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -28674,9 +27935,9 @@ _loop0_48_rule(Parser *p) return _seq; } -// _loop0_49: param_maybe_default +// _loop0_40: '|' closed_pattern static asdl_seq * -_loop0_49_rule(Parser *p) +_loop0_40_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28696,18 +27957,27 @@ _loop0_49_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // param_maybe_default + { // '|' closed_pattern if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; + D(fprintf(stderr, "%*c> _loop0_40[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); + Token * _literal; + pattern_ty elem; while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default + (_literal = _PyPegen_expect_token(p, 18)) // token='|' + && + (elem = closed_pattern_rule(p)) // closed_pattern ) { - _res = param_maybe_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28724,8 +27994,8 @@ _loop0_49_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_49[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c%s _loop0_40[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'|' closed_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -28741,9 +28011,9 @@ _loop0_49_rule(Parser *p) return _seq; } -// _loop1_50: param_maybe_default +// _gather_41: closed_pattern _loop0_40 static asdl_seq * -_loop1_50_rule(Parser *p) +_gather_41_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28752,70 +28022,172 @@ _loop1_50_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); + { // closed_pattern _loop0_40 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_40")); + pattern_ty elem; + asdl_seq * seq; + if ( + (elem = closed_pattern_rule(p)) // closed_pattern + && + (seq = _loop0_40_rule(p)) // _loop0_40 + ) + { + D(fprintf(stderr, "%*c+ _gather_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_40")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_41[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "closed_pattern _loop0_40")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_42: '+' | '-' +static void * +_tmp_42_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { p->level--; return NULL; } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_maybe_default + void * _res = NULL; + int _mark = p->mark; + { // '+' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_50[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; - while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - _res = param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_50[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + { // '-' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 15)) // token='-' + ) + { + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_43: '.' | '(' | '=' +static void * +_tmp_43_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { p->level--; return NULL; } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + void * _res = NULL; + int _mark = p->mark; + { // '.' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 23)) // token='.' + ) + { + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + } + { // '(' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 7)) // token='(' + ) + { + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); + } + { // '=' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + ) + { + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); + } + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop0_52: ',' with_item +// _loop0_44: ',' maybe_star_pattern static asdl_seq * -_loop0_52_rule(Parser *p) +_loop0_44_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28835,18 +28207,18 @@ _loop0_52_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' with_item + { // ',' maybe_star_pattern if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c> _loop0_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); Token * _literal; - withitem_ty elem; + pattern_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = with_item_rule(p)) // with_item + (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern ) { _res = elem; @@ -28872,8 +28244,8 @@ _loop0_52_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_52[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c%s _loop0_44[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' maybe_star_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -28889,9 +28261,9 @@ _loop0_52_rule(Parser *p) return _seq; } -// _gather_51: with_item _loop0_52 +// _gather_45: maybe_star_pattern _loop0_44 static asdl_seq * -_gather_51_rule(Parser *p) +_gather_45_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28902,27 +28274,27 @@ _gather_51_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // with_item _loop0_52 + { // maybe_star_pattern _loop0_44 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_52")); - withitem_ty elem; + D(fprintf(stderr, "%*c> _gather_45[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_44")); + pattern_ty elem; asdl_seq * seq; if ( - (elem = with_item_rule(p)) // with_item + (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern && - (seq = _loop0_52_rule(p)) // _loop0_52 + (seq = _loop0_44_rule(p)) // _loop0_44 ) { - D(fprintf(stderr, "%*c+ _gather_51[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_52")); + D(fprintf(stderr, "%*c+ _gather_45[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_44")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_51[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_52")); + D(fprintf(stderr, "%*c%s _gather_45[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_44")); } _res = NULL; done: @@ -28930,9 +28302,9 @@ _gather_51_rule(Parser *p) return _res; } -// _loop0_54: ',' with_item +// _loop0_46: ',' key_value_pattern static asdl_seq * -_loop0_54_rule(Parser *p) +_loop0_46_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -28952,18 +28324,18 @@ _loop0_54_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' with_item + { // ',' key_value_pattern if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c> _loop0_46[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); Token * _literal; - withitem_ty elem; + KeyPatternPair* elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = with_item_rule(p)) // with_item + (elem = key_value_pattern_rule(p)) // key_value_pattern ) { _res = elem; @@ -28989,8 +28361,8 @@ _loop0_54_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_54[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c%s _loop0_46[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' key_value_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -29006,9 +28378,9 @@ _loop0_54_rule(Parser *p) return _seq; } -// _gather_53: with_item _loop0_54 +// _gather_47: key_value_pattern _loop0_46 static asdl_seq * -_gather_53_rule(Parser *p) +_gather_47_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29019,27 +28391,84 @@ _gather_53_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // with_item _loop0_54 + { // key_value_pattern _loop0_46 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_53[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_54")); - withitem_ty elem; + D(fprintf(stderr, "%*c> _gather_47[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_46")); + KeyPatternPair* elem; asdl_seq * seq; if ( - (elem = with_item_rule(p)) // with_item + (elem = key_value_pattern_rule(p)) // key_value_pattern && - (seq = _loop0_54_rule(p)) // _loop0_54 + (seq = _loop0_46_rule(p)) // _loop0_46 ) { - D(fprintf(stderr, "%*c+ _gather_53[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_54")); + D(fprintf(stderr, "%*c+ _gather_47[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_46")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_53[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_54")); + D(fprintf(stderr, "%*c%s _gather_47[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "key_value_pattern _loop0_46")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_48: literal_expr | attr +static void * +_tmp_48_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // literal_expr + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); + expr_ty literal_expr_var; + if ( + (literal_expr_var = literal_expr_rule(p)) // literal_expr + ) + { + D(fprintf(stderr, "%*c+ _tmp_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); + _res = literal_expr_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_48[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "literal_expr")); + } + { // attr + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); + expr_ty attr_var; + if ( + (attr_var = attr_rule(p)) // attr + ) + { + D(fprintf(stderr, "%*c+ _tmp_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); + _res = attr_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_48[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "attr")); } _res = NULL; done: @@ -29047,9 +28476,9 @@ _gather_53_rule(Parser *p) return _res; } -// _loop0_56: ',' with_item +// _loop0_49: ',' pattern static asdl_seq * -_loop0_56_rule(Parser *p) +_loop0_49_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29069,18 +28498,18 @@ _loop0_56_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' with_item + { // ',' pattern if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c> _loop0_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); Token * _literal; - withitem_ty elem; + pattern_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = with_item_rule(p)) // with_item + (elem = pattern_rule(p)) // pattern ) { _res = elem; @@ -29106,8 +28535,8 @@ _loop0_56_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_56[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c%s _loop0_49[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -29123,9 +28552,9 @@ _loop0_56_rule(Parser *p) return _seq; } -// _gather_55: with_item _loop0_56 +// _gather_50: pattern _loop0_49 static asdl_seq * -_gather_55_rule(Parser *p) +_gather_50_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29136,27 +28565,27 @@ _gather_55_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // with_item _loop0_56 + { // pattern _loop0_49 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_56")); - withitem_ty elem; + D(fprintf(stderr, "%*c> _gather_50[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_49")); + pattern_ty elem; asdl_seq * seq; if ( - (elem = with_item_rule(p)) // with_item + (elem = pattern_rule(p)) // pattern && - (seq = _loop0_56_rule(p)) // _loop0_56 + (seq = _loop0_49_rule(p)) // _loop0_49 ) { - D(fprintf(stderr, "%*c+ _gather_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_56")); + D(fprintf(stderr, "%*c+ _gather_50[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern _loop0_49")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_55[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_56")); + D(fprintf(stderr, "%*c%s _gather_50[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern _loop0_49")); } _res = NULL; done: @@ -29164,9 +28593,9 @@ _gather_55_rule(Parser *p) return _res; } -// _loop0_58: ',' with_item +// _loop0_51: ',' keyword_pattern static asdl_seq * -_loop0_58_rule(Parser *p) +_loop0_51_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29186,18 +28615,18 @@ _loop0_58_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' with_item + { // ',' keyword_pattern if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c> _loop0_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); Token * _literal; - withitem_ty elem; + KeyPatternPair* elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = with_item_rule(p)) // with_item + (elem = keyword_pattern_rule(p)) // keyword_pattern ) { _res = elem; @@ -29223,8 +28652,8 @@ _loop0_58_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_58[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c%s _loop0_51[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' keyword_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -29240,9 +28669,9 @@ _loop0_58_rule(Parser *p) return _seq; } -// _gather_57: with_item _loop0_58 +// _gather_52: keyword_pattern _loop0_51 static asdl_seq * -_gather_57_rule(Parser *p) +_gather_52_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29253,27 +28682,27 @@ _gather_57_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // with_item _loop0_58 + { // keyword_pattern _loop0_51 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_58")); - withitem_ty elem; + D(fprintf(stderr, "%*c> _gather_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_51")); + KeyPatternPair* elem; asdl_seq * seq; if ( - (elem = with_item_rule(p)) // with_item + (elem = keyword_pattern_rule(p)) // keyword_pattern && - (seq = _loop0_58_rule(p)) // _loop0_58 + (seq = _loop0_51_rule(p)) // _loop0_51 ) { - D(fprintf(stderr, "%*c+ _gather_57[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_58")); + D(fprintf(stderr, "%*c+ _gather_52[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_51")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_57[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_58")); + D(fprintf(stderr, "%*c%s _gather_52[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "keyword_pattern _loop0_51")); } _res = NULL; done: @@ -29281,9 +28710,9 @@ _gather_57_rule(Parser *p) return _res; } -// _tmp_59: ',' | ')' | ':' -static void * -_tmp_59_rule(Parser *p) +// _loop0_53: ',' type_param +static asdl_seq * +_loop0_53_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29292,64 +28721,105 @@ _tmp_59_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // ',' + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' type_param if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _loop0_53[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); Token * _literal; - if ( + type_param_ty elem; + while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = type_param_rule(p)) // type_param ) { - D(fprintf(stderr, "%*c+ _tmp_59[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_59[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + D(fprintf(stderr, "%*c%s _loop0_53[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' type_param")); } - { // ')' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 8)) // token=')' - ) - { - D(fprintf(stderr, "%*c+ _tmp_59[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_59[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; } - { // ':' + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _gather_54: type_param _loop0_53 +static asdl_seq * +_gather_54_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // type_param _loop0_53 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; + D(fprintf(stderr, "%*c> _gather_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_53")); + type_param_ty elem; + asdl_seq * seq; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (elem = type_param_rule(p)) // type_param + && + (seq = _loop0_53_rule(p)) // _loop0_53 ) { - D(fprintf(stderr, "%*c+ _tmp_59[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _gather_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "type_param _loop0_53")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_59[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _gather_54[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "type_param _loop0_53")); } _res = NULL; done: @@ -29357,9 +28827,9 @@ _tmp_59_rule(Parser *p) return _res; } -// _loop1_60: except_block +// _loop1_55: (',' expression) static asdl_seq * -_loop1_60_rule(Parser *p) +_loop1_55_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29379,18 +28849,18 @@ _loop1_60_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // except_block + { // (',' expression) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); - excepthandler_ty except_block_var; + D(fprintf(stderr, "%*c> _loop1_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); + void *_tmp_17_var; while ( - (except_block_var = except_block_rule(p)) // except_block + (_tmp_17_var = _tmp_17_rule(p)) // ',' expression ) { - _res = except_block_var; + _res = _tmp_17_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29407,8 +28877,8 @@ _loop1_60_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_60[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c%s _loop1_55[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' expression)")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -29429,9 +28899,9 @@ _loop1_60_rule(Parser *p) return _seq; } -// _loop1_61: except_star_block +// _loop1_56: (',' star_expression) static asdl_seq * -_loop1_61_rule(Parser *p) +_loop1_56_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29451,18 +28921,18 @@ _loop1_61_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // except_star_block + { // (',' star_expression) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); - excepthandler_ty except_star_block_var; + D(fprintf(stderr, "%*c> _loop1_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); + void *_tmp_153_var; while ( - (except_star_block_var = except_star_block_rule(p)) // except_star_block + (_tmp_153_var = _tmp_153_rule(p)) // ',' star_expression ) { - _res = except_star_block_var; + _res = _tmp_153_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29479,8 +28949,8 @@ _loop1_61_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_61[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c%s _loop1_56[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -29501,9 +28971,9 @@ _loop1_61_rule(Parser *p) return _seq; } -// _tmp_62: 'as' NAME -static void * -_tmp_62_rule(Parser *p) +// _loop0_57: ',' star_named_expression +static asdl_seq * +_loop0_57_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29512,44 +28982,74 @@ _tmp_62_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // 'as' NAME + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' star_named_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty z; - if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' + D(fprintf(stderr, "%*c> _loop0_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); + Token * _literal; + expr_ty elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (z = _PyPegen_name_token(p)) // NAME + (elem = star_named_expression_rule(p)) // star_named_expression ) { - D(fprintf(stderr, "%*c+ _tmp_62[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = z; + _res = elem; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; + PyMem_Free(_children); p->level--; return NULL; } - goto done; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_62[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _loop0_57[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); } - _res = NULL; - done: + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); p->level--; - return _res; + return _seq; } -// _tmp_63: 'as' NAME -static void * -_tmp_63_rule(Parser *p) +// _gather_58: star_named_expression _loop0_57 +static asdl_seq * +_gather_58_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29558,34 +29058,29 @@ _tmp_63_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // star_named_expression _loop0_57 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty z; + D(fprintf(stderr, "%*c> _gather_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_57")); + expr_ty elem; + asdl_seq * seq; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' + (elem = star_named_expression_rule(p)) // star_named_expression && - (z = _PyPegen_name_token(p)) // NAME + (seq = _loop0_57_rule(p)) // _loop0_57 ) { - D(fprintf(stderr, "%*c+ _tmp_63[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _gather_58[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_57")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_63[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _gather_58[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_57")); } _res = NULL; done: @@ -29593,9 +29088,9 @@ _tmp_63_rule(Parser *p) return _res; } -// _loop1_64: case_block +// _loop1_59: ('or' conjunction) static asdl_seq * -_loop1_64_rule(Parser *p) +_loop1_59_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29615,18 +29110,18 @@ _loop1_64_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // case_block + { // ('or' conjunction) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "case_block")); - match_case_ty case_block_var; + D(fprintf(stderr, "%*c> _loop1_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); + void *_tmp_154_var; while ( - (case_block_var = case_block_rule(p)) // case_block + (_tmp_154_var = _tmp_154_rule(p)) // 'or' conjunction ) { - _res = case_block_var; + _res = _tmp_154_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29643,8 +29138,8 @@ _loop1_64_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_64[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "case_block")); + D(fprintf(stderr, "%*c%s _loop1_59[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('or' conjunction)")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -29665,9 +29160,9 @@ _loop1_64_rule(Parser *p) return _seq; } -// _loop0_66: '|' closed_pattern +// _loop1_60: ('and' inversion) static asdl_seq * -_loop0_66_rule(Parser *p) +_loop1_60_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29687,27 +29182,18 @@ _loop0_66_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // '|' closed_pattern + { // ('and' inversion) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); - Token * _literal; - pattern_ty elem; + D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); + void *_tmp_155_var; while ( - (_literal = _PyPegen_expect_token(p, 18)) // token='|' - && - (elem = closed_pattern_rule(p)) // closed_pattern + (_tmp_155_var = _tmp_155_rule(p)) // 'and' inversion ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } + _res = _tmp_155_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -29724,8 +29210,13 @@ _loop0_66_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_66[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'|' closed_pattern")); + D(fprintf(stderr, "%*c%s _loop1_60[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('and' inversion)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -29741,9 +29232,9 @@ _loop0_66_rule(Parser *p) return _seq; } -// _gather_65: closed_pattern _loop0_66 +// _loop1_61: compare_op_bitwise_or_pair static asdl_seq * -_gather_65_rule(Parser *p) +_loop1_61_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29752,39 +29243,70 @@ _gather_65_rule(Parser *p) p->level--; return NULL; } - asdl_seq * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // closed_pattern _loop0_66 + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // compare_op_bitwise_or_pair if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_66")); - pattern_ty elem; - asdl_seq * seq; - if ( - (elem = closed_pattern_rule(p)) // closed_pattern - && - (seq = _loop0_66_rule(p)) // _loop0_66 + D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compare_op_bitwise_or_pair")); + CmpopExprPair* compare_op_bitwise_or_pair_var; + while ( + (compare_op_bitwise_or_pair_var = compare_op_bitwise_or_pair_rule(p)) // compare_op_bitwise_or_pair ) { - D(fprintf(stderr, "%*c+ _gather_65[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_66")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_65[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "closed_pattern _loop0_66")); - } - _res = NULL; - done: - p->level--; - return _res; + _res = compare_op_bitwise_or_pair_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_61[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compare_op_bitwise_or_pair")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; } -// _tmp_67: '+' | '-' +// _tmp_62: '!=' static void * -_tmp_67_rule(Parser *p) +_tmp_62_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29795,43 +29317,29 @@ _tmp_67_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '+' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - ) - { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); - } - { // '-' + { // '!=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!='")); + Token * tok; if ( - (_literal = _PyPegen_expect_token(p, 15)) // token='-' + (tok = _PyPegen_expect_token(p, 28)) // token='!=' ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_62[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); + _res = _PyPegen_check_barry_as_flufl ( p , tok ) ? NULL : tok; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c%s _tmp_62[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!='")); } _res = NULL; done: @@ -29839,9 +29347,9 @@ _tmp_67_rule(Parser *p) return _res; } -// _tmp_68: '+' | '-' -static void * -_tmp_68_rule(Parser *p) +// _loop0_63: ',' (slice | starred_expression) +static asdl_seq * +_loop0_63_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29850,45 +29358,105 @@ _tmp_68_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // '+' + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' (slice | starred_expression) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _loop0_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (slice | starred_expression)")); Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' + void *elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_156_rule(p)) // slice | starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); - _res = _literal; - goto done; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c%s _loop0_63[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (slice | starred_expression)")); } - { // '-' + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _gather_64: (slice | starred_expression) _loop0_63 +static asdl_seq * +_gather_64_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // (slice | starred_expression) _loop0_63 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); - Token * _literal; + D(fprintf(stderr, "%*c> _gather_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_63")); + void *elem; + asdl_seq * seq; if ( - (_literal = _PyPegen_expect_token(p, 15)) // token='-' + (elem = _tmp_156_rule(p)) // slice | starred_expression + && + (seq = _loop0_63_rule(p)) // _loop0_63 ) { - D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _gather_64[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_63")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c%s _gather_64[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(slice | starred_expression) _loop0_63")); } _res = NULL; done: @@ -29896,9 +29464,9 @@ _tmp_68_rule(Parser *p) return _res; } -// _tmp_69: '.' | '(' | '=' +// _tmp_65: ':' expression? static void * -_tmp_69_rule(Parser *p) +_tmp_65_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29909,62 +29477,89 @@ _tmp_69_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '.' + { // ':' expression? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression?")); Token * _literal; + void *d; if ( - (_literal = _PyPegen_expect_token(p, 23)) // token='.' + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + && + (d = expression_rule(p), !p->error_indicator) // expression? ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_65[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); + _res = d; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c%s _tmp_65[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':' expression?")); } - { // '(' + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_66: STRING | FSTRING_START +static void * +_tmp_66_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // STRING if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "STRING")); + expr_ty string_var; if ( - (_literal = _PyPegen_expect_token(p, 7)) // token='(' + (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); + _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "STRING")); } - { // '=' + { // FSTRING_START if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); + Token * fstring_start_var; if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' + (fstring_start_var = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); + _res = fstring_start_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "FSTRING_START")); } _res = NULL; done: @@ -29972,9 +29567,9 @@ _tmp_69_rule(Parser *p) return _res; } -// _tmp_70: '.' | '(' | '=' +// _tmp_67: tuple | group | genexp static void * -_tmp_70_rule(Parser *p) +_tmp_67_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -29985,62 +29580,62 @@ _tmp_70_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '.' + { // tuple if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + expr_ty tuple_var; if ( - (_literal = _PyPegen_expect_token(p, 23)) // token='.' + (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } - { // '(' + { // group if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group")); + expr_ty group_var; if ( - (_literal = _PyPegen_expect_token(p, 7)) // token='(' + (group_var = group_rule(p)) // group ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "group")); + _res = group_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "group")); } - { // '=' + { // genexp if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + expr_ty genexp_var; if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' + (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } _res = NULL; done: @@ -30048,9 +29643,9 @@ _tmp_70_rule(Parser *p) return _res; } -// _loop0_72: ',' maybe_star_pattern -static asdl_seq * -_loop0_72_rule(Parser *p) +// _tmp_68: list | listcomp +static void * +_tmp_68_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30059,105 +29654,45 @@ _loop0_72_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' maybe_star_pattern + { // list if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_72[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); - Token * _literal; - pattern_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + expr_ty list_var; + if ( + (list_var = list_rule(p)) // list ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + _res = list_var; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_72[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' maybe_star_pattern")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_71: maybe_star_pattern _loop0_72 -static asdl_seq * -_gather_71_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // maybe_star_pattern _loop0_72 + { // listcomp if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_71[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_72")); - pattern_ty elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "listcomp")); + expr_ty listcomp_var; if ( - (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern - && - (seq = _loop0_72_rule(p)) // _loop0_72 + (listcomp_var = listcomp_rule(p)) // listcomp ) { - D(fprintf(stderr, "%*c+ _gather_71[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_72")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "listcomp")); + _res = listcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_71[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_72")); + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "listcomp")); } _res = NULL; done: @@ -30165,9 +29700,9 @@ _gather_71_rule(Parser *p) return _res; } -// _loop0_74: ',' key_value_pattern -static asdl_seq * -_loop0_74_rule(Parser *p) +// _tmp_69: dict | set | dictcomp | setcomp +static void * +_tmp_69_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30176,105 +29711,83 @@ _loop0_74_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' key_value_pattern + { // dict if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_74[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); - Token * _literal; - KeyPatternPair* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = key_value_pattern_rule(p)) // key_value_pattern + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dict")); + expr_ty dict_var; + if ( + (dict_var = dict_rule(p)) // dict ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dict")); + _res = dict_var; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_74[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' key_value_pattern")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dict")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_73: key_value_pattern _loop0_74 -static asdl_seq * -_gather_73_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); + { // set + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "set")); + expr_ty set_var; + if ( + (set_var = set_rule(p)) // set + ) + { + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "set")); + _res = set_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "set")); } - if (p->error_indicator) { - p->level--; - return NULL; + { // dictcomp + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dictcomp")); + expr_ty dictcomp_var; + if ( + (dictcomp_var = dictcomp_rule(p)) // dictcomp + ) + { + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dictcomp")); + _res = dictcomp_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dictcomp")); } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // key_value_pattern _loop0_74 + { // setcomp if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_73[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_74")); - KeyPatternPair* elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "setcomp")); + expr_ty setcomp_var; if ( - (elem = key_value_pattern_rule(p)) // key_value_pattern - && - (seq = _loop0_74_rule(p)) // _loop0_74 + (setcomp_var = setcomp_rule(p)) // setcomp ) { - D(fprintf(stderr, "%*c+ _gather_73[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_74")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "setcomp")); + _res = setcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_73[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "key_value_pattern _loop0_74")); + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "setcomp")); } _res = NULL; done: @@ -30282,9 +29795,9 @@ _gather_73_rule(Parser *p) return _res; } -// _tmp_75: literal_expr | attr +// _tmp_70: yield_expr | named_expression static void * -_tmp_75_rule(Parser *p) +_tmp_70_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30295,43 +29808,43 @@ _tmp_75_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // literal_expr + { // yield_expr if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); - expr_ty literal_expr_var; + D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + expr_ty yield_expr_var; if ( - (literal_expr_var = literal_expr_rule(p)) // literal_expr + (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_75[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); - _res = literal_expr_var; + D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_75[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "literal_expr")); + D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } - { // attr + { // named_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); - expr_ty attr_var; + D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "named_expression")); + expr_ty named_expression_var; if ( - (attr_var = attr_rule(p)) // attr + (named_expression_var = named_expression_rule(p)) // named_expression ) { - D(fprintf(stderr, "%*c+ _tmp_75[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); - _res = attr_var; + D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); + _res = named_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_75[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "attr")); + D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "named_expression")); } _res = NULL; done: @@ -30339,9 +29852,9 @@ _tmp_75_rule(Parser *p) return _res; } -// _loop0_77: ',' pattern +// _loop0_71: lambda_param_no_default static asdl_seq * -_loop0_77_rule(Parser *p) +_loop0_71_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30361,27 +29874,18 @@ _loop0_77_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' pattern + { // lambda_param_no_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_77[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); - Token * _literal; - pattern_ty elem; + D(fprintf(stderr, "%*c> _loop0_71[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + arg_ty lambda_param_no_default_var; while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = pattern_rule(p)) // pattern + (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } + _res = lambda_param_no_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30398,8 +29902,8 @@ _loop0_77_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_77[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' pattern")); + D(fprintf(stderr, "%*c%s _loop0_71[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -30415,9 +29919,9 @@ _loop0_77_rule(Parser *p) return _seq; } -// _gather_76: pattern _loop0_77 +// _loop0_72: lambda_param_with_default static asdl_seq * -_gather_76_rule(Parser *p) +_loop0_72_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30426,48 +29930,7 @@ _gather_76_rule(Parser *p) p->level--; return NULL; } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // pattern _loop0_77 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_76[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_77")); - pattern_ty elem; - asdl_seq * seq; - if ( - (elem = pattern_rule(p)) // pattern - && - (seq = _loop0_77_rule(p)) // _loop0_77 - ) - { - D(fprintf(stderr, "%*c+ _gather_76[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern _loop0_77")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_76[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern _loop0_77")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_79: ',' keyword_pattern -static asdl_seq * -_loop0_79_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; + void *_res = NULL; int _mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { @@ -30478,27 +29941,18 @@ _loop0_79_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' keyword_pattern + { // lambda_param_with_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_79[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); - Token * _literal; - KeyPatternPair* elem; + D(fprintf(stderr, "%*c> _loop0_72[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + NameDefaultPair* lambda_param_with_default_var; while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = keyword_pattern_rule(p)) // keyword_pattern + (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } + _res = lambda_param_with_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30515,8 +29969,8 @@ _loop0_79_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_79[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' keyword_pattern")); + D(fprintf(stderr, "%*c%s _loop0_72[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -30532,50 +29986,9 @@ _loop0_79_rule(Parser *p) return _seq; } -// _gather_78: keyword_pattern _loop0_79 +// _loop1_73: lambda_param_no_default static asdl_seq * -_gather_78_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // keyword_pattern _loop0_79 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_78[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_79")); - KeyPatternPair* elem; - asdl_seq * seq; - if ( - (elem = keyword_pattern_rule(p)) // keyword_pattern - && - (seq = _loop0_79_rule(p)) // _loop0_79 - ) - { - D(fprintf(stderr, "%*c+ _gather_78[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_79")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_78[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "keyword_pattern _loop0_79")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_81: ',' type_param -static asdl_seq * -_loop0_81_rule(Parser *p) +_loop1_73_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30595,27 +30008,18 @@ _loop0_81_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' type_param + { // lambda_param_no_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); - Token * _literal; - type_param_ty elem; + D(fprintf(stderr, "%*c> _loop1_73[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + arg_ty lambda_param_no_default_var; while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = type_param_rule(p)) // type_param + (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } + _res = lambda_param_no_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30632,8 +30036,13 @@ _loop0_81_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_81[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' type_param")); + D(fprintf(stderr, "%*c%s _loop1_73[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -30649,50 +30058,9 @@ _loop0_81_rule(Parser *p) return _seq; } -// _gather_80: type_param _loop0_81 +// _loop1_74: lambda_param_with_default static asdl_seq * -_gather_80_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // type_param _loop0_81 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_81")); - type_param_ty elem; - asdl_seq * seq; - if ( - (elem = type_param_rule(p)) // type_param - && - (seq = _loop0_81_rule(p)) // _loop0_81 - ) - { - D(fprintf(stderr, "%*c+ _gather_80[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "type_param _loop0_81")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_80[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "type_param _loop0_81")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_82: (',' expression) -static asdl_seq * -_loop1_82_rule(Parser *p) +_loop1_74_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30712,18 +30080,18 @@ _loop1_82_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // (',' expression) + { // lambda_param_with_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_251_var; + D(fprintf(stderr, "%*c> _loop1_74[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + NameDefaultPair* lambda_param_with_default_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // ',' expression + (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default ) { - _res = _tmp_251_var; + _res = lambda_param_with_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30740,8 +30108,8 @@ _loop1_82_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_82[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' expression)")); + D(fprintf(stderr, "%*c%s _loop1_74[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -30762,9 +30130,9 @@ _loop1_82_rule(Parser *p) return _seq; } -// _loop1_83: (',' star_expression) +// _loop0_75: lambda_param_maybe_default static asdl_seq * -_loop1_83_rule(Parser *p) +_loop0_75_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30784,18 +30152,18 @@ _loop1_83_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // (',' star_expression) + { // lambda_param_maybe_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_252_var; + D(fprintf(stderr, "%*c> _loop0_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + NameDefaultPair* lambda_param_maybe_default_var; while ( - (_tmp_252_var = _tmp_252_rule(p)) // ',' star_expression + (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default ) { - _res = _tmp_252_var; + _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30812,13 +30180,8 @@ _loop1_83_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_83[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_75[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -30834,9 +30197,9 @@ _loop1_83_rule(Parser *p) return _seq; } -// _loop0_85: ',' star_named_expression +// _loop1_76: lambda_param_maybe_default static asdl_seq * -_loop0_85_rule(Parser *p) +_loop1_76_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30856,27 +30219,18 @@ _loop0_85_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' star_named_expression + { // lambda_param_maybe_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); - Token * _literal; - expr_ty elem; + D(fprintf(stderr, "%*c> _loop1_76[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + NameDefaultPair* lambda_param_maybe_default_var; while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = star_named_expression_rule(p)) // star_named_expression + (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } + _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30893,8 +30247,13 @@ _loop0_85_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_85[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); + D(fprintf(stderr, "%*c%s _loop1_76[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -30910,50 +30269,9 @@ _loop0_85_rule(Parser *p) return _seq; } -// _gather_84: star_named_expression _loop0_85 -static asdl_seq * -_gather_84_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // star_named_expression _loop0_85 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_85")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = star_named_expression_rule(p)) // star_named_expression - && - (seq = _loop0_85_rule(p)) // _loop0_85 - ) - { - D(fprintf(stderr, "%*c+ _gather_84[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_85")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_84[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_85")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_86: ('or' conjunction) +// _loop0_77: fstring_format_spec static asdl_seq * -_loop1_86_rule(Parser *p) +_loop0_77_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -30973,18 +30291,18 @@ _loop1_86_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ('or' conjunction) + { // fstring_format_spec if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_253_var; + D(fprintf(stderr, "%*c> _loop0_77[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + expr_ty fstring_format_spec_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // 'or' conjunction + (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec ) { - _res = _tmp_253_var; + _res = fstring_format_spec_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31001,13 +30319,8 @@ _loop1_86_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_86[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('or' conjunction)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_77[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31023,9 +30336,9 @@ _loop1_86_rule(Parser *p) return _seq; } -// _loop1_87: ('and' inversion) +// _loop0_78: fstring_middle static asdl_seq * -_loop1_87_rule(Parser *p) +_loop0_78_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31045,18 +30358,18 @@ _loop1_87_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ('and' inversion) + { // fstring_middle if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_254_var; + D(fprintf(stderr, "%*c> _loop0_78[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); + expr_ty fstring_middle_var; while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'and' inversion + (fstring_middle_var = fstring_middle_rule(p)) // fstring_middle ) { - _res = _tmp_254_var; + _res = fstring_middle_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31073,13 +30386,8 @@ _loop1_87_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_87[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('and' inversion)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_78[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_middle")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31095,9 +30403,9 @@ _loop1_87_rule(Parser *p) return _seq; } -// _loop1_88: compare_op_bitwise_or_pair +// _loop1_79: (fstring | string) static asdl_seq * -_loop1_88_rule(Parser *p) +_loop1_79_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31117,18 +30425,18 @@ _loop1_88_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // compare_op_bitwise_or_pair + { // (fstring | string) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compare_op_bitwise_or_pair")); - CmpopExprPair* compare_op_bitwise_or_pair_var; + D(fprintf(stderr, "%*c> _loop1_79[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); + void *_tmp_157_var; while ( - (compare_op_bitwise_or_pair_var = compare_op_bitwise_or_pair_rule(p)) // compare_op_bitwise_or_pair + (_tmp_157_var = _tmp_157_rule(p)) // fstring | string ) { - _res = compare_op_bitwise_or_pair_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31145,8 +30453,8 @@ _loop1_88_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_88[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compare_op_bitwise_or_pair")); + D(fprintf(stderr, "%*c%s _loop1_79[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -31167,9 +30475,9 @@ _loop1_88_rule(Parser *p) return _seq; } -// _tmp_89: '!=' +// _tmp_80: star_named_expression ',' star_named_expressions? static void * -_tmp_89_rule(Parser *p) +_tmp_80_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31180,19 +30488,25 @@ _tmp_89_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '!=' + { // star_named_expression ',' star_named_expressions? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_89[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!='")); - Token * tok; + D(fprintf(stderr, "%*c> _tmp_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + Token * _literal; + expr_ty y; + void *z; if ( - (tok = _PyPegen_expect_token(p, 28)) // token='!=' + (y = star_named_expression_rule(p)) // star_named_expression + && + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { - D(fprintf(stderr, "%*c+ _tmp_89[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); - _res = _PyPegen_check_barry_as_flufl ( p , tok ) ? NULL : tok; + D(fprintf(stderr, "%*c+ _tmp_80[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + _res = _PyPegen_seq_insert_in_front ( p , y , z ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -31201,8 +30515,8 @@ _tmp_89_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_89[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!='")); + D(fprintf(stderr, "%*c%s _tmp_80[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression ',' star_named_expressions?")); } _res = NULL; done: @@ -31210,9 +30524,9 @@ _tmp_89_rule(Parser *p) return _res; } -// _loop0_91: ',' (slice | starred_expression) +// _loop0_81: ',' double_starred_kvpair static asdl_seq * -_loop0_91_rule(Parser *p) +_loop0_81_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31232,18 +30546,18 @@ _loop0_91_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' (slice | starred_expression) + { // ',' double_starred_kvpair if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_91[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (slice | starred_expression)")); + D(fprintf(stderr, "%*c> _loop0_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; - void *elem; + KeyValuePair* elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_255_rule(p)) // slice | starred_expression + (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair ) { _res = elem; @@ -31269,8 +30583,8 @@ _loop0_91_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_91[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (slice | starred_expression)")); + D(fprintf(stderr, "%*c%s _loop0_81[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31286,9 +30600,9 @@ _loop0_91_rule(Parser *p) return _seq; } -// _gather_90: (slice | starred_expression) _loop0_91 +// _gather_82: double_starred_kvpair _loop0_81 static asdl_seq * -_gather_90_rule(Parser *p) +_gather_82_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31299,27 +30613,27 @@ _gather_90_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (slice | starred_expression) _loop0_91 + { // double_starred_kvpair _loop0_81 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_90[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_91")); - void *elem; + D(fprintf(stderr, "%*c> _gather_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_81")); + KeyValuePair* elem; asdl_seq * seq; if ( - (elem = _tmp_255_rule(p)) // slice | starred_expression + (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_91_rule(p)) // _loop0_91 + (seq = _loop0_81_rule(p)) // _loop0_81 ) { - D(fprintf(stderr, "%*c+ _gather_90[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_91")); + D(fprintf(stderr, "%*c+ _gather_82[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_81")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_90[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(slice | starred_expression) _loop0_91")); + D(fprintf(stderr, "%*c%s _gather_82[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_81")); } _res = NULL; done: @@ -31327,9 +30641,9 @@ _gather_90_rule(Parser *p) return _res; } -// _tmp_92: ':' expression? -static void * -_tmp_92_rule(Parser *p) +// _loop1_83: for_if_clause +static asdl_seq * +_loop1_83_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31338,44 +30652,70 @@ _tmp_92_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // ':' expression? + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // for_if_clause if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_92[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression?")); - Token * _literal; - void *d; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - && - (d = expression_rule(p), !p->error_indicator) // expression? + D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); + comprehension_ty for_if_clause_var; + while ( + (for_if_clause_var = for_if_clause_rule(p)) // for_if_clause ) { - D(fprintf(stderr, "%*c+ _tmp_92[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); - _res = d; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; + _res = for_if_clause_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; } - goto done; + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_92[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':' expression?")); + D(fprintf(stderr, "%*c%s _loop1_83[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "for_if_clause")); } - _res = NULL; - done: + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); p->level--; - return _res; + return _seq; } -// _tmp_93: STRING | FSTRING_START -static void * -_tmp_93_rule(Parser *p) +// _loop0_84: ('if' disjunction) +static asdl_seq * +_loop0_84_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31384,55 +30724,65 @@ _tmp_93_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // STRING - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "STRING")); - expr_ty string_var; - if ( - (string_var = _PyPegen_string_token(p)) // STRING - ) - { - D(fprintf(stderr, "%*c+ _tmp_93[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); - _res = string_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_93[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "STRING")); + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; } - { // FSTRING_START + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ('if' disjunction) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); - Token * fstring_start_var; - if ( - (fstring_start_var = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' + D(fprintf(stderr, "%*c> _loop0_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); + void *_tmp_158_var; + while ( + (_tmp_158_var = _tmp_158_rule(p)) // 'if' disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_93[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); - _res = fstring_start_var; - goto done; + _res = _tmp_158_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_93[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "FSTRING_START")); + D(fprintf(stderr, "%*c%s _loop0_84[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); } - _res = NULL; - done: + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); p->level--; - return _res; + return _seq; } -// _tmp_94: tuple | group | genexp +// _tmp_85: assignment_expression | expression !':=' static void * -_tmp_94_rule(Parser *p) +_tmp_85_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31443,62 +30793,45 @@ _tmp_94_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // tuple - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); - expr_ty tuple_var; - if ( - (tuple_var = tuple_rule(p)) // tuple - ) - { - D(fprintf(stderr, "%*c+ _tmp_94[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); - _res = tuple_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_94[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); - } - { // group + { // assignment_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group")); - expr_ty group_var; + D(fprintf(stderr, "%*c> _tmp_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + expr_ty assignment_expression_var; if ( - (group_var = group_rule(p)) // group + (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_94[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "group")); - _res = group_var; + D(fprintf(stderr, "%*c+ _tmp_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_94[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "group")); + D(fprintf(stderr, "%*c%s _tmp_85[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } - { // genexp + { // expression !':=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); - expr_ty genexp_var; + D(fprintf(stderr, "%*c> _tmp_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + expr_ty expression_var; if ( - (genexp_var = genexp_rule(p)) // genexp + (expression_var = expression_rule(p)) // expression + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_94[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); - _res = genexp_var; + D(fprintf(stderr, "%*c+ _tmp_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_94[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c%s _tmp_85[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; done: @@ -31506,9 +30839,9 @@ _tmp_94_rule(Parser *p) return _res; } -// _tmp_95: list | listcomp -static void * -_tmp_95_rule(Parser *p) +// _loop0_86: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +static asdl_seq * +_loop0_86_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31517,55 +30850,75 @@ _tmp_95_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // list - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); - expr_ty list_var; - if ( - (list_var = list_rule(p)) // list - ) - { - D(fprintf(stderr, "%*c+ _tmp_95[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); - _res = list_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_95[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; } - { // listcomp + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "listcomp")); - expr_ty listcomp_var; - if ( - (listcomp_var = listcomp_rule(p)) // listcomp + D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + Token * _literal; + void *elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_159_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { - D(fprintf(stderr, "%*c+ _tmp_95[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "listcomp")); - _res = listcomp_var; - goto done; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_95[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "listcomp")); + D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } - _res = NULL; - done: + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); p->level--; - return _res; + return _seq; } -// _tmp_96: dict | set | dictcomp | setcomp -static void * -_tmp_96_rule(Parser *p) +// _gather_87: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_86 +static asdl_seq * +_gather_87_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31574,83 +30927,29 @@ _tmp_96_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - { // dict - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dict")); - expr_ty dict_var; - if ( - (dict_var = dict_rule(p)) // dict - ) - { - D(fprintf(stderr, "%*c+ _tmp_96[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dict")); - _res = dict_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_96[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dict")); - } - { // set - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "set")); - expr_ty set_var; - if ( - (set_var = set_rule(p)) // set - ) - { - D(fprintf(stderr, "%*c+ _tmp_96[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "set")); - _res = set_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_96[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "set")); - } - { // dictcomp - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dictcomp")); - expr_ty dictcomp_var; - if ( - (dictcomp_var = dictcomp_rule(p)) // dictcomp - ) - { - D(fprintf(stderr, "%*c+ _tmp_96[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dictcomp")); - _res = dictcomp_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_96[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dictcomp")); - } - { // setcomp + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_86 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "setcomp")); - expr_ty setcomp_var; + D(fprintf(stderr, "%*c> _gather_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_86")); + void *elem; + asdl_seq * seq; if ( - (setcomp_var = setcomp_rule(p)) // setcomp + (elem = _tmp_159_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + && + (seq = _loop0_86_rule(p)) // _loop0_86 ) { - D(fprintf(stderr, "%*c+ _tmp_96[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "setcomp")); - _res = setcomp_var; + D(fprintf(stderr, "%*c+ _gather_87[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_86")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_96[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "setcomp")); + D(fprintf(stderr, "%*c%s _gather_87[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_86")); } _res = NULL; done: @@ -31658,9 +30957,9 @@ _tmp_96_rule(Parser *p) return _res; } -// _tmp_97: yield_expr | named_expression +// _tmp_88: ',' kwargs static void * -_tmp_97_rule(Parser *p) +_tmp_88_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31671,43 +30970,32 @@ _tmp_97_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // yield_expr - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); - expr_ty yield_expr_var; - if ( - (yield_expr_var = yield_expr_rule(p)) // yield_expr - ) - { - D(fprintf(stderr, "%*c+ _tmp_97[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); - _res = yield_expr_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_97[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); - } - { // named_expression + { // ',' kwargs if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "named_expression")); - expr_ty named_expression_var; + D(fprintf(stderr, "%*c> _tmp_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + Token * _literal; + asdl_seq* k; if ( - (named_expression_var = named_expression_rule(p)) // named_expression + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (k = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_97[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); - _res = named_expression_var; + D(fprintf(stderr, "%*c+ _tmp_88[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + _res = k; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_97[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "named_expression")); + D(fprintf(stderr, "%*c%s _tmp_88[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); } _res = NULL; done: @@ -31715,9 +31003,9 @@ _tmp_97_rule(Parser *p) return _res; } -// _loop0_98: lambda_param_no_default +// _loop0_89: ',' kwarg_or_starred static asdl_seq * -_loop0_98_rule(Parser *p) +_loop0_89_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31737,18 +31025,27 @@ _loop0_98_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_no_default + { // ',' kwarg_or_starred if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; + D(fprintf(stderr, "%*c> _loop0_89[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + Token * _literal; + KeywordOrStarred* elem; while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred ) { - _res = lambda_param_no_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31765,8 +31062,8 @@ _loop0_98_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_98[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c%s _loop0_89[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31782,9 +31079,50 @@ _loop0_98_rule(Parser *p) return _seq; } -// _loop0_99: lambda_param_with_default +// _gather_90: kwarg_or_starred _loop0_89 +static asdl_seq * +_gather_90_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // kwarg_or_starred _loop0_89 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_90[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_89")); + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred + && + (seq = _loop0_89_rule(p)) // _loop0_89 + ) + { + D(fprintf(stderr, "%*c+ _gather_90[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_89")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_90[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_89")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop0_91: ',' kwarg_or_double_starred static asdl_seq * -_loop0_99_rule(Parser *p) +_loop0_91_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31804,18 +31142,27 @@ _loop0_99_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_with_default + { // ',' kwarg_or_double_starred if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_99[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; + D(fprintf(stderr, "%*c> _loop0_91[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + Token * _literal; + KeywordOrStarred* elem; while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred ) { - _res = lambda_param_with_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31832,8 +31179,8 @@ _loop0_99_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_99[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c%s _loop0_91[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31849,9 +31196,50 @@ _loop0_99_rule(Parser *p) return _seq; } -// _loop0_100: lambda_param_with_default +// _gather_92: kwarg_or_double_starred _loop0_91 static asdl_seq * -_loop0_100_rule(Parser *p) +_gather_92_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + asdl_seq * _res = NULL; + int _mark = p->mark; + { // kwarg_or_double_starred _loop0_91 + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _gather_92[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_91")); + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred + && + (seq = _loop0_91_rule(p)) // _loop0_91 + ) + { + D(fprintf(stderr, "%*c+ _gather_92[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_91")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _gather_92[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_91")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop0_93: (',' star_target) +static asdl_seq * +_loop0_93_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31871,18 +31259,18 @@ _loop0_100_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_with_default + { // (',' star_target) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_100[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; + D(fprintf(stderr, "%*c> _loop0_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_160_var; while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default + (_tmp_160_var = _tmp_160_rule(p)) // ',' star_target ) { - _res = lambda_param_with_default_var; + _res = _tmp_160_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31899,8 +31287,8 @@ _loop0_100_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_100[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c%s _loop0_93[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31916,9 +31304,9 @@ _loop0_100_rule(Parser *p) return _seq; } -// _loop1_101: lambda_param_no_default +// _loop0_94: ',' star_target static asdl_seq * -_loop1_101_rule(Parser *p) +_loop0_94_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31938,18 +31326,27 @@ _loop1_101_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_no_default + { // ',' star_target if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_101[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; + D(fprintf(stderr, "%*c> _loop0_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + Token * _literal; + expr_ty elem; while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = star_target_rule(p)) // star_target ) { - _res = lambda_param_no_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31966,13 +31363,8 @@ _loop1_101_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_101[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_94[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -31988,9 +31380,9 @@ _loop1_101_rule(Parser *p) return _seq; } -// _loop0_102: lambda_param_with_default +// _gather_95: star_target _loop0_94 static asdl_seq * -_loop0_102_rule(Parser *p) +_gather_95_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -31999,65 +31391,39 @@ _loop0_102_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_with_default + { // star_target _loop0_94 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_102[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; - while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default + D(fprintf(stderr, "%*c> _gather_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_94")); + expr_ty elem; + asdl_seq * seq; + if ( + (elem = star_target_rule(p)) // star_target + && + (seq = _loop0_94_rule(p)) // _loop0_94 ) { - _res = lambda_param_with_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _gather_95[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_94")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_102[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _gather_95[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_94")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop1_103: lambda_param_with_default +// _loop1_96: (',' star_target) static asdl_seq * -_loop1_103_rule(Parser *p) +_loop1_96_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32077,18 +31443,18 @@ _loop1_103_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_with_default + { // (',' star_target) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; + D(fprintf(stderr, "%*c> _loop1_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_160_var; while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default + (_tmp_160_var = _tmp_160_rule(p)) // ',' star_target ) { - _res = lambda_param_with_default_var; + _res = _tmp_160_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32105,8 +31471,8 @@ _loop1_103_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_103[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c%s _loop1_96[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -32127,9 +31493,9 @@ _loop1_103_rule(Parser *p) return _seq; } -// _loop1_104: lambda_param_no_default -static asdl_seq * -_loop1_104_rule(Parser *p) +// _tmp_97: !'*' star_target +static void * +_tmp_97_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32138,70 +31504,38 @@ _loop1_104_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default + { // !'*' star_target if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_104[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + D(fprintf(stderr, "%*c> _tmp_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + expr_ty star_target_var; + if ( + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' + && + (star_target_var = star_target_rule(p)) // star_target ) { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_97[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + _res = star_target_var; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_104[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_97[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop1_105: lambda_param_no_default +// _loop0_98: ',' del_target static asdl_seq * -_loop1_105_rule(Parser *p) +_loop0_98_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32221,18 +31555,27 @@ _loop1_105_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_no_default + { // ',' del_target if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; + D(fprintf(stderr, "%*c> _loop0_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); + Token * _literal; + expr_ty elem; while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = del_target_rule(p)) // del_target ) { - _res = lambda_param_no_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32249,13 +31592,8 @@ _loop1_105_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_105[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_98[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -32271,9 +31609,9 @@ _loop1_105_rule(Parser *p) return _seq; } -// _loop0_106: lambda_param_no_default +// _gather_99: del_target _loop0_98 static asdl_seq * -_loop0_106_rule(Parser *p) +_gather_99_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32282,65 +31620,39 @@ _loop0_106_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default + { // del_target _loop0_98 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + D(fprintf(stderr, "%*c> _gather_99[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_98")); + expr_ty elem; + asdl_seq * seq; + if ( + (elem = del_target_rule(p)) // del_target + && + (seq = _loop0_98_rule(p)) // _loop0_98 ) { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _gather_99[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_98")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_106[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _gather_99[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_98")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop1_107: lambda_param_with_default +// _loop0_100: ',' expression static asdl_seq * -_loop1_107_rule(Parser *p) +_loop0_100_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32360,18 +31672,27 @@ _loop1_107_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_with_default + { // ',' expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_107[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; + D(fprintf(stderr, "%*c> _loop0_100[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + Token * _literal; + expr_ty elem; while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = expression_rule(p)) // expression ) { - _res = lambda_param_with_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32388,13 +31709,8 @@ _loop1_107_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_107[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_100[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -32410,9 +31726,9 @@ _loop1_107_rule(Parser *p) return _seq; } -// _loop0_108: lambda_param_no_default +// _gather_101: expression _loop0_100 static asdl_seq * -_loop0_108_rule(Parser *p) +_gather_101_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32421,65 +31737,39 @@ _loop0_108_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default + { // expression _loop0_100 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + D(fprintf(stderr, "%*c> _gather_101[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_100")); + expr_ty elem; + asdl_seq * seq; + if ( + (elem = expression_rule(p)) // expression + && + (seq = _loop0_100_rule(p)) // _loop0_100 ) { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _gather_101[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_100")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_108[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _gather_101[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_100")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop1_109: lambda_param_with_default -static asdl_seq * -_loop1_109_rule(Parser *p) +// _tmp_102: NEWLINE INDENT +static void * +_tmp_102_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32488,70 +31778,41 @@ _loop1_109_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_with_default + { // NEWLINE INDENT if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; - while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default + D(fprintf(stderr, "%*c> _tmp_102[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + Token * indent_var; + Token * newline_var; + if ( + (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' + && + (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' ) { - _res = lambda_param_with_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_102[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + _res = _PyPegen_dummy_name(p, newline_var, indent_var); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_109[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_102[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop0_110: lambda_param_maybe_default -static asdl_seq * -_loop0_110_rule(Parser *p) +// _tmp_103: +// | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) +// | kwargs +static void * +_tmp_103_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32560,65 +31821,55 @@ _loop0_110_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_maybe_default + { // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; - while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default + D(fprintf(stderr, "%*c> _tmp_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + void *_tmp_161_var; + if ( + (_tmp_161_var = _tmp_161_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { - _res = lambda_param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_103[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + _res = _tmp_161_var; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_110[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c%s _tmp_103[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + { // kwargs + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); + asdl_seq* kwargs_var; + if ( + (kwargs_var = kwargs_rule(p)) // kwargs + ) + { + D(fprintf(stderr, "%*c+ _tmp_103[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); + _res = kwargs_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_103[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop1_111: lambda_param_maybe_default +// _loop0_104: ',' (starred_expression !'=') static asdl_seq * -_loop1_111_rule(Parser *p) +_loop0_104_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32638,18 +31889,27 @@ _loop1_111_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // lambda_param_maybe_default + { // ',' (starred_expression !'=') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; + D(fprintf(stderr, "%*c> _loop0_104[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); + Token * _literal; + void *elem; while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_162_rule(p)) // starred_expression !'=' ) { - _res = lambda_param_maybe_default_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32666,13 +31926,8 @@ _loop1_111_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_111[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_104[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -32688,9 +31943,9 @@ _loop1_111_rule(Parser *p) return _seq; } -// _loop0_112: fstring_format_spec +// _gather_105: (starred_expression !'=') _loop0_104 static asdl_seq * -_loop0_112_rule(Parser *p) +_gather_105_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32699,65 +31954,39 @@ _loop0_112_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // fstring_format_spec + { // (starred_expression !'=') _loop0_104 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); - expr_ty fstring_format_spec_var; - while ( - (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec + D(fprintf(stderr, "%*c> _gather_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_104")); + void *elem; + asdl_seq * seq; + if ( + (elem = _tmp_162_rule(p)) // starred_expression !'=' + && + (seq = _loop0_104_rule(p)) // _loop0_104 ) { - _res = fstring_format_spec_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _gather_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_104")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_112[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _gather_105[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_104")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _loop0_113: fstring_middle -static asdl_seq * -_loop0_113_rule(Parser *p) +// _tmp_106: args | expression for_if_clauses +static void * +_tmp_106_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -32766,5375 +31995,58 @@ _loop0_113_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // fstring_middle + { // args if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); - expr_ty fstring_middle_var; - while ( - (fstring_middle_var = fstring_middle_rule(p)) // fstring_middle + D(fprintf(stderr, "%*c> _tmp_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + expr_ty args_var; + if ( + (args_var = args_rule(p)) // args ) { - _res = fstring_middle_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_106[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + _res = args_var; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_113[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_middle")); + D(fprintf(stderr, "%*c%s _tmp_106[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_114: (fstring | string) -static asdl_seq * -_loop1_114_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (fstring | string) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_256_var; - while ( - (_tmp_256_var = _tmp_256_rule(p)) // fstring | string - ) - { - _res = _tmp_256_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_114[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_115: star_named_expression ',' star_named_expressions? -static void * -_tmp_115_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // star_named_expression ',' star_named_expressions? - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); - Token * _literal; - expr_ty y; - void *z; - if ( - (y = star_named_expression_rule(p)) // star_named_expression - && - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? - ) - { - D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); - _res = _PyPegen_seq_insert_in_front ( p , y , z ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression ',' star_named_expressions?")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_117: ',' double_starred_kvpair -static asdl_seq * -_loop0_117_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' double_starred_kvpair - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); - Token * _literal; - KeyValuePair* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_117[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_116: double_starred_kvpair _loop0_117 -static asdl_seq * -_gather_116_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // double_starred_kvpair _loop0_117 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_117")); - KeyValuePair* elem; - asdl_seq * seq; - if ( - (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair - && - (seq = _loop0_117_rule(p)) // _loop0_117 - ) - { - D(fprintf(stderr, "%*c+ _gather_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_117")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_116[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_117")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_118: for_if_clause -static asdl_seq * -_loop1_118_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // for_if_clause - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); - comprehension_ty for_if_clause_var; - while ( - (for_if_clause_var = for_if_clause_rule(p)) // for_if_clause - ) - { - _res = for_if_clause_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_118[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "for_if_clause")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_119: ('if' disjunction) -static asdl_seq * -_loop0_119_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ('if' disjunction) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_257_var; - while ( - (_tmp_257_var = _tmp_257_rule(p)) // 'if' disjunction - ) - { - _res = _tmp_257_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_119[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_120: ('if' disjunction) -static asdl_seq * -_loop0_120_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ('if' disjunction) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_258_var; - while ( - (_tmp_258_var = _tmp_258_rule(p)) // 'if' disjunction - ) - { - _res = _tmp_258_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_121: assignment_expression | expression !':=' -static void * -_tmp_121_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // assignment_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); - expr_ty assignment_expression_var; - if ( - (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); - _res = assignment_expression_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); - } - { // expression !':=' + { // expression for_if_clauses if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; + asdl_comprehension_seq* for_if_clauses_var; if ( (expression_var = expression_rule(p)) // expression && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' + (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); - _res = expression_var; + D(fprintf(stderr, "%*c+ _tmp_106[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c%s _tmp_106[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; done: p->level--; - return _res; -} - -// _loop0_123: ',' (starred_expression | (assignment_expression | expression !':=') !'=') -static asdl_seq * -_loop0_123_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); - Token * _literal; - void *elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_259_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_123[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_122: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123 -static asdl_seq * -_gather_122_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); - void *elem; - asdl_seq * seq; - if ( - (elem = _tmp_259_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' - && - (seq = _loop0_123_rule(p)) // _loop0_123 - ) - { - D(fprintf(stderr, "%*c+ _gather_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_122[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_124: ',' kwargs -static void * -_tmp_124_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' kwargs - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); - Token * _literal; - asdl_seq* k; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (k = kwargs_rule(p)) // kwargs - ) - { - D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); - _res = k; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_126: ',' kwarg_or_starred -static asdl_seq * -_loop0_126_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' kwarg_or_starred - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); - Token * _literal; - KeywordOrStarred* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_126[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_125: kwarg_or_starred _loop0_126 -static asdl_seq * -_gather_125_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // kwarg_or_starred _loop0_126 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_126")); - KeywordOrStarred* elem; - asdl_seq * seq; - if ( - (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred - && - (seq = _loop0_126_rule(p)) // _loop0_126 - ) - { - D(fprintf(stderr, "%*c+ _gather_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_126")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_125[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_126")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_128: ',' kwarg_or_double_starred -static asdl_seq * -_loop0_128_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' kwarg_or_double_starred - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); - Token * _literal; - KeywordOrStarred* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_128[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_127: kwarg_or_double_starred _loop0_128 -static asdl_seq * -_gather_127_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // kwarg_or_double_starred _loop0_128 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_128")); - KeywordOrStarred* elem; - asdl_seq * seq; - if ( - (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred - && - (seq = _loop0_128_rule(p)) // _loop0_128 - ) - { - D(fprintf(stderr, "%*c+ _gather_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_128")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_127[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_128")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_130: ',' kwarg_or_starred -static asdl_seq * -_loop0_130_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' kwarg_or_starred - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); - Token * _literal; - KeywordOrStarred* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_130[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_129: kwarg_or_starred _loop0_130 -static asdl_seq * -_gather_129_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // kwarg_or_starred _loop0_130 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_130")); - KeywordOrStarred* elem; - asdl_seq * seq; - if ( - (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred - && - (seq = _loop0_130_rule(p)) // _loop0_130 - ) - { - D(fprintf(stderr, "%*c+ _gather_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_130")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_129[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_130")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_132: ',' kwarg_or_double_starred -static asdl_seq * -_loop0_132_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' kwarg_or_double_starred - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); - Token * _literal; - KeywordOrStarred* elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_132[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_131: kwarg_or_double_starred _loop0_132 -static asdl_seq * -_gather_131_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // kwarg_or_double_starred _loop0_132 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_132")); - KeywordOrStarred* elem; - asdl_seq * seq; - if ( - (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred - && - (seq = _loop0_132_rule(p)) // _loop0_132 - ) - { - D(fprintf(stderr, "%*c+ _gather_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_132")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_131[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_132")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_133: (',' star_target) -static asdl_seq * -_loop0_133_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (',' star_target) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_260_var; - while ( - (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target - ) - { - _res = _tmp_260_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_133[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_135: ',' star_target -static asdl_seq * -_loop0_135_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' star_target - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = star_target_rule(p)) // star_target - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_135[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_134: star_target _loop0_135 -static asdl_seq * -_gather_134_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // star_target _loop0_135 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_135")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = star_target_rule(p)) // star_target - && - (seq = _loop0_135_rule(p)) // _loop0_135 - ) - { - D(fprintf(stderr, "%*c+ _gather_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_135")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_134[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_135")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_136: (',' star_target) -static asdl_seq * -_loop1_136_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (',' star_target) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_261_var; - while ( - (_tmp_261_var = _tmp_261_rule(p)) // ',' star_target - ) - { - _res = _tmp_261_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_136[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_137: !'*' star_target -static void * -_tmp_137_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // !'*' star_target - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); - expr_ty star_target_var; - if ( - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' - && - (star_target_var = star_target_rule(p)) // star_target - ) - { - D(fprintf(stderr, "%*c+ _tmp_137[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); - _res = star_target_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_137[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_139: ',' del_target -static asdl_seq * -_loop0_139_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' del_target - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = del_target_rule(p)) // del_target - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_139[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_138: del_target _loop0_139 -static asdl_seq * -_gather_138_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // del_target _loop0_139 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_139")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = del_target_rule(p)) // del_target - && - (seq = _loop0_139_rule(p)) // _loop0_139 - ) - { - D(fprintf(stderr, "%*c+ _gather_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_139")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_138[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_139")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_141: ',' expression -static asdl_seq * -_loop0_141_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = expression_rule(p)) // expression - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_141[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_140: expression _loop0_141 -static asdl_seq * -_gather_140_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // expression _loop0_141 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_141")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = expression_rule(p)) // expression - && - (seq = _loop0_141_rule(p)) // _loop0_141 - ) - { - D(fprintf(stderr, "%*c+ _gather_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_141")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_140[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_141")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_143: ',' expression -static asdl_seq * -_loop0_143_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = expression_rule(p)) // expression - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_143[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_142: expression _loop0_143 -static asdl_seq * -_gather_142_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // expression _loop0_143 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = expression_rule(p)) // expression - && - (seq = _loop0_143_rule(p)) // _loop0_143 - ) - { - D(fprintf(stderr, "%*c+ _gather_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_142[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_143")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_145: ',' expression -static asdl_seq * -_loop0_145_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = expression_rule(p)) // expression - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_145[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_144: expression _loop0_145 -static asdl_seq * -_gather_144_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // expression _loop0_145 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = expression_rule(p)) // expression - && - (seq = _loop0_145_rule(p)) // _loop0_145 - ) - { - D(fprintf(stderr, "%*c+ _gather_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_144[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_145")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_147: ',' expression -static asdl_seq * -_loop0_147_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = expression_rule(p)) // expression - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_147[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_146: expression _loop0_147 -static asdl_seq * -_gather_146_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // expression _loop0_147 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); - expr_ty elem; - asdl_seq * seq; - if ( - (elem = expression_rule(p)) // expression - && - (seq = _loop0_147_rule(p)) // _loop0_147 - ) - { - D(fprintf(stderr, "%*c+ _gather_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_146[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_147")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_148: NEWLINE INDENT -static void * -_tmp_148_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // NEWLINE INDENT - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); - Token * indent_var; - Token * newline_var; - if ( - (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' - && - (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' - ) - { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); - _res = _PyPegen_dummy_name(p, newline_var, indent_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_149: -// | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) -// | kwargs -static void * -_tmp_149_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_262_var; - if ( - (_tmp_262_var = _tmp_262_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs - ) - { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_262_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - } - { // kwargs - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); - asdl_seq* kwargs_var; - if ( - (kwargs_var = kwargs_rule(p)) // kwargs - ) - { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); - _res = kwargs_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_151: ',' (starred_expression !'=') -static asdl_seq * -_loop0_151_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' (starred_expression !'=') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); - Token * _literal; - void *elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_263_rule(p)) // starred_expression !'=' - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_151[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_150: (starred_expression !'=') _loop0_151 -static asdl_seq * -_gather_150_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // (starred_expression !'=') _loop0_151 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_151")); - void *elem; - asdl_seq * seq; - if ( - (elem = _tmp_263_rule(p)) // starred_expression !'=' - && - (seq = _loop0_151_rule(p)) // _loop0_151 - ) - { - D(fprintf(stderr, "%*c+ _gather_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_151")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_150[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_151")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_152: args | expression for_if_clauses -static void * -_tmp_152_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // args - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); - expr_ty args_var; - if ( - (args_var = args_rule(p)) // args - ) - { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); - _res = args_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); - } - { // expression for_if_clauses - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); - expr_ty expression_var; - asdl_comprehension_seq* for_if_clauses_var; - if ( - (expression_var = expression_rule(p)) // expression - && - (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses - ) - { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); - _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_153: args ',' -static void * -_tmp_153_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // args ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); - Token * _literal; - expr_ty args_var; - if ( - (args_var = args_rule(p)) // args - && - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); - _res = _PyPegen_dummy_name(p, args_var, _literal); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_154: ',' | ')' -static void * -_tmp_154_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - { // ')' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 8)) // token=')' - ) - { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_155: 'True' | 'False' | 'None' -static void * -_tmp_155_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'True' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' - ) - { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); - } - { // 'False' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' - ) - { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); - } - { // 'None' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' - ) - { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_156: NAME '=' -static void * -_tmp_156_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // NAME '=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); - Token * _literal; - expr_ty name_var; - if ( - (name_var = _PyPegen_name_token(p)) // NAME - && - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); - _res = _PyPegen_dummy_name(p, name_var, _literal); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_157: NAME STRING | SOFT_KEYWORD -static void * -_tmp_157_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // NAME STRING - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); - expr_ty name_var; - expr_ty string_var; - if ( - (name_var = _PyPegen_name_token(p)) // NAME - && - (string_var = _PyPegen_string_token(p)) // STRING - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); - _res = _PyPegen_dummy_name(p, name_var, string_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); - } - { // SOFT_KEYWORD - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); - expr_ty soft_keyword_var; - if ( - (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); - _res = soft_keyword_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_158: 'else' | ':' -static void * -_tmp_158_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'else' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 671)) // token='else' - ) - { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); - } - { // ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_159: '=' | ':=' -static void * -_tmp_159_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); - } - { // ':=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 53)) // token=':=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_160: list | tuple | genexp | 'True' | 'None' | 'False' -static void * -_tmp_160_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // list - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); - expr_ty list_var; - if ( - (list_var = list_rule(p)) // list - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); - _res = list_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); - } - { // tuple - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); - expr_ty tuple_var; - if ( - (tuple_var = tuple_rule(p)) // tuple - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); - _res = tuple_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); - } - { // genexp - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); - expr_ty genexp_var; - if ( - (genexp_var = genexp_rule(p)) // genexp - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); - _res = genexp_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); - } - { // 'True' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 610)) // token='True' - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); - } - { // 'None' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 611)) // token='None' - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); - } - { // 'False' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); - Token * _keyword; - if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='False' - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); - _res = _keyword; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_161: '=' | ':=' -static void * -_tmp_161_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); - } - { // ':=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 53)) // token=':=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_162: star_named_expressions -static asdl_seq * -_loop0_162_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // star_named_expressions - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); - asdl_expr_seq* star_named_expressions_var; - while ( - (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions - ) - { - _res = star_named_expressions_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_163: (star_targets '=') -static asdl_seq * -_loop0_163_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (star_targets '=') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_264_var; - while ( - (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' - ) - { - _res = _tmp_264_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_164: (star_targets '=') -static asdl_seq * -_loop0_164_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (star_targets '=') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_265_var; - while ( - (_tmp_265_var = _tmp_265_rule(p)) // star_targets '=' - ) - { - _res = _tmp_265_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_165: '[' | '(' | '{' -static void * -_tmp_165_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '[' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 9)) // token='[' - ) - { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); - } - { // '(' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 7)) // token='(' - ) - { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); - } - { // '{' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 25)) // token='{' - ) - { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_166: '[' | '{' -static void * -_tmp_166_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '[' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 9)) // token='[' - ) - { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); - } - { // '{' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 25)) // token='{' - ) - { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_167: '[' | '{' -static void * -_tmp_167_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '[' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 9)) // token='[' - ) - { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); - } - { // '{' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 25)) // token='{' - ) - { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_168: slash_no_default | slash_with_default -static void * -_tmp_168_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // slash_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); - asdl_arg_seq* slash_no_default_var; - if ( - (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); - _res = slash_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); - } - { // slash_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); - SlashWithDefault* slash_with_default_var; - if ( - (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); - _res = slash_with_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_169: param_maybe_default -static asdl_seq * -_loop0_169_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; - while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default - ) - { - _res = param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_170: param_no_default -static asdl_seq * -_loop0_170_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - _res = param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_171: param_no_default -static asdl_seq * -_loop0_171_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - _res = param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_172: param_no_default -static asdl_seq * -_loop1_172_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - _res = param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_172[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_173: slash_no_default | slash_with_default -static void * -_tmp_173_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // slash_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); - asdl_arg_seq* slash_no_default_var; - if ( - (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); - _res = slash_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); - } - { // slash_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); - SlashWithDefault* slash_with_default_var; - if ( - (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); - _res = slash_with_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_174: param_maybe_default -static asdl_seq * -_loop0_174_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; - while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default - ) - { - _res = param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_174[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_175: ',' | param_no_default -static void * -_tmp_175_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - if ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); - _res = param_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_176: param_maybe_default -static asdl_seq * -_loop0_176_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; - while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default - ) - { - _res = param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_176[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_177: param_maybe_default -static asdl_seq * -_loop1_177_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; - while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default - ) - { - _res = param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_177[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_178: ')' | ',' -static void * -_tmp_178_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ')' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 8)) // token=')' - ) - { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); - } - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_179: ')' | ',' (')' | '**') -static void * -_tmp_179_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ')' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 8)) // token=')' - ) - { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); - } - { // ',' (')' | '**') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - Token * _literal; - void *_tmp_266_var; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (_tmp_266_var = _tmp_266_rule(p)) // ')' | '**' - ) - { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_180: param_no_default | ',' -static void * -_tmp_180_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - if ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); - _res = param_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_181: param_maybe_default -static asdl_seq * -_loop0_181_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); - NameDefaultPair* param_maybe_default_var; - while ( - (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default - ) - { - _res = param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_181[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_182: param_no_default | ',' -static void * -_tmp_182_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - if ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); - _res = param_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_183: '*' | '**' | '/' -static void * -_tmp_183_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '*' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' - ) - { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); - } - { // '**' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' - ) - { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); - } - { // '/' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 17)) // token='/' - ) - { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop1_184: param_with_default -static asdl_seq * -_loop1_184_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); - NameDefaultPair* param_with_default_var; - while ( - (param_with_default_var = param_with_default_rule(p)) // param_with_default - ) - { - _res = param_with_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_184[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_185: lambda_slash_no_default | lambda_slash_with_default -static void * -_tmp_185_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // lambda_slash_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); - asdl_arg_seq* lambda_slash_no_default_var; - if ( - (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); - _res = lambda_slash_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); - } - { // lambda_slash_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); - SlashWithDefault* lambda_slash_with_default_var; - if ( - (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); - _res = lambda_slash_with_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_186: lambda_param_maybe_default -static asdl_seq * -_loop0_186_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; - while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default - ) - { - _res = lambda_param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_187: lambda_param_no_default -static asdl_seq * -_loop0_187_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_188: lambda_param_no_default -static asdl_seq * -_loop0_188_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_190: ',' lambda_param -static asdl_seq * -_loop0_190_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' lambda_param - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); - Token * _literal; - arg_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = lambda_param_rule(p)) // lambda_param - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_190[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_189: lambda_param _loop0_190 -static asdl_seq * -_gather_189_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // lambda_param _loop0_190 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); - arg_ty elem; - asdl_seq * seq; - if ( - (elem = lambda_param_rule(p)) // lambda_param - && - (seq = _loop0_190_rule(p)) // _loop0_190 - ) - { - D(fprintf(stderr, "%*c+ _gather_189[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_189[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_190")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_191: lambda_slash_no_default | lambda_slash_with_default -static void * -_tmp_191_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // lambda_slash_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); - asdl_arg_seq* lambda_slash_no_default_var; - if ( - (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); - _res = lambda_slash_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); - } - { // lambda_slash_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); - SlashWithDefault* lambda_slash_with_default_var; - if ( - (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); - _res = lambda_slash_with_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_192: lambda_param_maybe_default -static asdl_seq * -_loop0_192_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; - while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default - ) - { - _res = lambda_param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_192[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_193: ',' | lambda_param_no_default -static void * -_tmp_193_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - if ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - _res = lambda_param_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_194: lambda_param_maybe_default -static asdl_seq * -_loop0_194_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; - while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default - ) - { - _res = lambda_param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_194[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_195: lambda_param_maybe_default -static asdl_seq * -_loop1_195_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; - while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default - ) - { - _res = lambda_param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_196: lambda_param_with_default -static asdl_seq * -_loop1_196_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_with_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); - NameDefaultPair* lambda_param_with_default_var; - while ( - (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default - ) - { - _res = lambda_param_with_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_197: ':' | ',' (':' | '**') -static void * -_tmp_197_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); - } - { // ',' (':' | '**') - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - Token * _literal; - void *_tmp_267_var; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (_tmp_267_var = _tmp_267_rule(p)) // ':' | '**' - ) - { - D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_267_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_198: lambda_param_no_default | ',' -static void * -_tmp_198_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - if ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - _res = lambda_param_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_199: lambda_param_maybe_default -static asdl_seq * -_loop0_199_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_maybe_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); - NameDefaultPair* lambda_param_maybe_default_var; - while ( - (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default - ) - { - _res = lambda_param_maybe_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_199[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; + return _res; } -// _tmp_200: lambda_param_no_default | ',' +// _tmp_107: args ',' static void * -_tmp_200_rule(Parser *p) +_tmp_107_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38145,43 +32057,27 @@ _tmp_200_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - if ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - _res = lambda_param_no_default_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - { // ',' + { // args ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_107[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; + expr_ty args_var; if ( + (args_var = args_rule(p)) // args + && (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_107[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + D(fprintf(stderr, "%*c%s _tmp_107[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; done: @@ -38189,9 +32085,9 @@ _tmp_200_rule(Parser *p) return _res; } -// _tmp_201: '*' | '**' | '/' +// _tmp_108: ',' | ')' static void * -_tmp_201_rule(Parser *p) +_tmp_108_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38202,62 +32098,43 @@ _tmp_201_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '*' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' - ) - { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); - } - { // '**' + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c%s _tmp_108[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } - { // '/' + { // ')' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 17)) // token='/' + (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c%s _tmp_108[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; done: @@ -38265,9 +32142,9 @@ _tmp_201_rule(Parser *p) return _res; } -// _tmp_202: ',' | ')' | ':' +// _tmp_109: 'True' | 'False' | 'None' static void * -_tmp_202_rule(Parser *p) +_tmp_109_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38278,62 +32155,62 @@ _tmp_202_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ',' + { // 'True' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + Token * _keyword; if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } - { // ')' + { // 'False' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + Token * _keyword; if ( - (_literal = _PyPegen_expect_token(p, 8)) // token=')' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } - { // ':' + { // 'None' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + Token * _keyword; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; done: @@ -38341,9 +32218,9 @@ _tmp_202_rule(Parser *p) return _res; } -// _tmp_203: bitwise_or ((',' bitwise_or))* ','? +// _tmp_110: NAME '=' static void * -_tmp_203_rule(Parser *p) +_tmp_110_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38354,31 +32231,27 @@ _tmp_203_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // bitwise_or ((',' bitwise_or))* ','? + { // NAME '=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_268_var; - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty bitwise_or_var; + D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + Token * _literal; + expr_ty name_var; if ( - (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or - && - (_loop0_268_var = _loop0_268_rule(p)) // ((',' bitwise_or))* + (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? + (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_268_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; done: @@ -38386,9 +32259,9 @@ _tmp_203_rule(Parser *p) return _res; } -// _loop0_205: ',' dotted_name -static asdl_seq * -_loop0_205_rule(Parser *p) +// _tmp_111: NAME STRING | SOFT_KEYWORD +static void * +_tmp_111_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38397,105 +32270,48 @@ _loop0_205_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' dotted_name + { // NAME STRING if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); - Token * _literal; - expr_ty elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + expr_ty name_var; + expr_ty string_var; + if ( + (name_var = _PyPegen_name_token(p)) // NAME && - (elem = dotted_name_rule(p)) // dotted_name + (string_var = _PyPegen_string_token(p)) // STRING ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + _res = _PyPegen_dummy_name(p, name_var, string_var); + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_204: dotted_name _loop0_205 -static asdl_seq * -_gather_204_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // dotted_name _loop0_205 + { // SOFT_KEYWORD if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); - expr_ty elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + expr_ty soft_keyword_var; if ( - (elem = dotted_name_rule(p)) // dotted_name - && - (seq = _loop0_205_rule(p)) // _loop0_205 + (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _gather_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_205")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_204[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_205")); + D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; done: @@ -38503,9 +32319,9 @@ _gather_204_rule(Parser *p) return _res; } -// _loop0_207: ',' (expression ['as' star_target]) -static asdl_seq * -_loop0_207_rule(Parser *p) +// _tmp_112: 'else' | ':' +static void * +_tmp_112_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38514,105 +32330,45 @@ _loop0_207_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' (expression ['as' star_target]) + { // 'else' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); - Token * _literal; - void *elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_269_rule(p)) // expression ['as' star_target] + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + Token * _keyword; + if ( + (_keyword = _PyPegen_expect_token(p, 671)) // token='else' ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + _res = _keyword; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_207[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_206: (expression ['as' star_target]) _loop0_207 -static asdl_seq * -_gather_206_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_207 + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); - void *elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; if ( - (elem = _tmp_269_rule(p)) // expression ['as' star_target] - && - (seq = _loop0_207_rule(p)) // _loop0_207 + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _gather_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_206[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_207")); + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; done: @@ -38620,9 +32376,9 @@ _gather_206_rule(Parser *p) return _res; } -// _loop0_209: ',' (expressions ['as' star_target]) -static asdl_seq * -_loop0_209_rule(Parser *p) +// _tmp_113: '=' | ':=' +static void * +_tmp_113_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38631,105 +32387,45 @@ _loop0_209_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' (expressions ['as' star_target]) + { // '=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _tmp_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; - void *elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_270_rule(p)) // expressions ['as' star_target] + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_113[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + _res = _literal; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_209[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_208: (expressions ['as' star_target]) _loop0_209 -static asdl_seq * -_gather_208_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_113[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_209 + { // ':=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); - void *elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + Token * _literal; if ( - (elem = _tmp_270_rule(p)) // expressions ['as' star_target] - && - (seq = _loop0_209_rule(p)) // _loop0_209 + (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _gather_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_113[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_208[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_209")); + D(fprintf(stderr, "%*c%s _tmp_113[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; done: @@ -38737,85 +32433,9 @@ _gather_208_rule(Parser *p) return _res; } -// _loop0_211: ',' (expression ['as' star_target]) -static asdl_seq * -_loop0_211_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' (expression ['as' star_target]) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); - Token * _literal; - void *elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_271_rule(p)) // expression ['as' star_target] - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_211[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_210: (expression ['as' star_target]) _loop0_211 -static asdl_seq * -_gather_210_rule(Parser *p) +// _tmp_114: list | tuple | genexp | 'True' | 'None' | 'False' +static void * +_tmp_114_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38824,203 +32444,121 @@ _gather_210_rule(Parser *p) p->level--; return NULL; } - asdl_seq * _res = NULL; + void * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_211 + { // list if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); - void *elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + expr_ty list_var; if ( - (elem = _tmp_271_rule(p)) // expression ['as' star_target] - && - (seq = _loop0_211_rule(p)) // _loop0_211 + (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _gather_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_210[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_211")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_213: ',' (expressions ['as' star_target]) -static asdl_seq * -_loop0_213_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // ',' (expressions ['as' star_target]) + { // tuple if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); - Token * _literal; - void *elem; - while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_272_rule(p)) // expressions ['as' star_target] - ) - { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_213[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _gather_212: (expressions ['as' star_target]) _loop0_213 -static asdl_seq * -_gather_212_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + expr_ty tuple_var; + if ( + (tuple_var = tuple_rule(p)) // tuple + ) + { + D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + _res = tuple_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_213 + { // genexp if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); - void *elem; - asdl_seq * seq; + D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + expr_ty genexp_var; if ( - (elem = _tmp_272_rule(p)) // expressions ['as' star_target] - && - (seq = _loop0_213_rule(p)) // _loop0_213 + (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _gather_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); + D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_212[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_213")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_214: 'except' | 'finally' -static void * -_tmp_214_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); + D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } - if (p->error_indicator) { - p->level--; - return NULL; + { // 'True' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + Token * _keyword; + if ( + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' + ) + { + D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + _res = _keyword; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } - void * _res = NULL; - int _mark = p->mark; - { // 'except' + { // 'None' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 663)) // token='except' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } - { // 'finally' + { // 'False' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 659)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; done: @@ -39028,9 +32566,9 @@ _tmp_214_rule(Parser *p) return _res; } -// _loop0_215: block +// _loop0_115: star_named_expressions static asdl_seq * -_loop0_215_rule(Parser *p) +_loop0_115_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39050,18 +32588,18 @@ _loop0_215_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // block + { // star_named_expressions if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); - asdl_stmt_seq* block_var; + D(fprintf(stderr, "%*c> _loop0_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + asdl_expr_seq* star_named_expressions_var; while ( - (block_var = block_rule(p)) // block + (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions ) { - _res = block_var; + _res = star_named_expressions_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -39078,8 +32616,8 @@ _loop0_215_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_215[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); + D(fprintf(stderr, "%*c%s _loop0_115[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -39095,9 +32633,9 @@ _loop0_215_rule(Parser *p) return _seq; } -// _loop1_216: except_block +// _loop0_116: (star_targets '=') static asdl_seq * -_loop1_216_rule(Parser *p) +_loop0_116_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39117,18 +32655,18 @@ _loop1_216_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // except_block + { // (star_targets '=') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); - excepthandler_ty except_block_var; + D(fprintf(stderr, "%*c> _loop0_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_150_var; while ( - (except_block_var = except_block_rule(p)) // except_block + (_tmp_150_var = _tmp_150_rule(p)) // star_targets '=' ) { - _res = except_block_var; + _res = _tmp_150_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -39145,13 +32683,8 @@ _loop1_216_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_216[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _loop0_116[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -39167,9 +32700,9 @@ _loop1_216_rule(Parser *p) return _seq; } -// _tmp_217: 'as' NAME +// _tmp_117: '[' | '(' | '{' static void * -_tmp_217_rule(Parser *p) +_tmp_117_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39180,176 +32713,72 @@ _tmp_217_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // '[' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_218: block -static asdl_seq * -_loop0_218_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // block + { // '(' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); - asdl_stmt_seq* block_var; - while ( - (block_var = block_rule(p)) // block + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - _res = block_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + _res = _literal; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_218[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_219: except_star_block -static asdl_seq * -_loop1_219_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // except_star_block + { // '{' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); - excepthandler_ty except_star_block_var; - while ( - (except_star_block_var = except_star_block_rule(p)) // except_star_block + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - _res = except_star_block_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + _res = _literal; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_219[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _tmp_220: expression ['as' NAME] +// _tmp_118: '[' | '{' static void * -_tmp_220_rule(Parser *p) +_tmp_118_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39360,69 +32789,43 @@ _tmp_220_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // expression ['as' NAME] + { // '[' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty expression_var; + D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + Token * _literal; if ( - (expression_var = expression_rule(p)) // expression - && - (_opt_var = _tmp_273_rule(p), !p->error_indicator) // ['as' NAME] + (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); - _res = _PyPegen_dummy_name(p, expression_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_221: 'as' NAME -static void * -_tmp_221_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } - void * _res = NULL; - int _mark = p->mark; - { // 'as' NAME + { // '{' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; done: @@ -39430,9 +32833,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: 'as' NAME +// _tmp_119: slash_no_default | slash_with_default static void * -_tmp_222_rule(Parser *p) +_tmp_119_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39443,68 +32846,43 @@ _tmp_222_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // slash_no_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + asdl_arg_seq* slash_no_default_var; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_223: 'as' NAME -static void * -_tmp_223_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } - void * _res = NULL; - int _mark = p->mark; - { // 'as' NAME + { // slash_with_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + SlashWithDefault* slash_with_default_var; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; done: @@ -39512,9 +32890,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: 'as' NAME +// _tmp_120: ',' | param_no_default static void * -_tmp_224_rule(Parser *p) +_tmp_120_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39525,27 +32903,43 @@ _tmp_224_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_120[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _tmp_120[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + } + { // param_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + arg_ty param_no_default_var; + if ( + (param_no_default_var = param_no_default_rule(p)) // param_no_default + ) + { + D(fprintf(stderr, "%*c+ _tmp_120[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + _res = param_no_default_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_120[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; done: @@ -39553,9 +32947,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: NEWLINE | ':' +// _tmp_121: ')' | ',' static void * -_tmp_225_rule(Parser *p) +_tmp_121_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39566,43 +32960,43 @@ _tmp_225_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // NEWLINE + { // ')' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); - Token * newline_var; + D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + Token * _literal; if ( - (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' + (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); - _res = newline_var; + D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } - { // ':' + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; done: @@ -39610,9 +33004,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: 'as' NAME +// _tmp_122: ')' | ',' (')' | '**') static void * -_tmp_226_rule(Parser *p) +_tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39623,68 +33017,46 @@ _tmp_226_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // ')' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_227: 'as' NAME -static void * -_tmp_227_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } - void * _res = NULL; - int _mark = p->mark; - { // 'as' NAME + { // ',' (')' | '**') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + Token * _literal; + void *_tmp_163_var; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (name_var = _PyPegen_name_token(p)) // NAME + (_tmp_163_var = _tmp_163_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_163_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; done: @@ -39692,9 +33064,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: positional_patterns ',' +// _tmp_123: param_no_default | ',' static void * -_tmp_228_rule(Parser *p) +_tmp_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39705,27 +33077,43 @@ _tmp_228_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // positional_patterns ',' + { // param_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + arg_ty param_no_default_var; + if ( + (param_no_default_var = param_no_default_rule(p)) // param_no_default + ) + { + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + _res = param_no_default_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); + } + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; - asdl_pattern_seq* positional_patterns_var; if ( - (positional_patterns_var = positional_patterns_rule(p)) // positional_patterns - && (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); - _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; done: @@ -39733,9 +33121,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _tmp_229: '->' expression +// _tmp_124: '*' | '**' | '/' static void * -_tmp_229_rule(Parser *p) +_tmp_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39746,113 +33134,62 @@ _tmp_229_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '->' expression + { // '*' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; - expr_ty expression_var; if ( - (_literal = _PyPegen_expect_token(p, 51)) // token='->' - && - (expression_var = expression_rule(p)) // expression + (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); - _res = _PyPegen_dummy_name(p, _literal, expression_var); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_230: '->' expression -static void * -_tmp_230_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } - void * _res = NULL; - int _mark = p->mark; - { // '->' expression + { // '**' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; - expr_ty expression_var; if ( - (_literal = _PyPegen_expect_token(p, 51)) // token='->' - && - (expression_var = expression_rule(p)) // expression + (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); - _res = _PyPegen_dummy_name(p, _literal, expression_var); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_231: '(' arguments? ')' -static void * -_tmp_231_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } - void * _res = NULL; - int _mark = p->mark; - { // '(' arguments? ')' + { // '/' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; - Token * _literal_1; - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings if ( - (_literal = _PyPegen_expect_token(p, 7)) // token='(' - && - (_opt_var = arguments_rule(p), !p->error_indicator) // arguments? - && - (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' + (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); - _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; done: @@ -39860,9 +33197,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: '(' arguments? ')' +// _tmp_125: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_232_rule(Parser *p) +_tmp_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39873,31 +33210,43 @@ _tmp_232_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '(' arguments? ')' + { // lambda_slash_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + asdl_arg_seq* lambda_slash_no_default_var; + if ( + (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default + ) + { + D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + _res = lambda_slash_no_default_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); + } + { // lambda_slash_with_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); - Token * _literal; - Token * _literal_1; - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings + D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + SlashWithDefault* lambda_slash_with_default_var; if ( - (_literal = _PyPegen_expect_token(p, 7)) // token='(' - && - (_opt_var = arguments_rule(p), !p->error_indicator) // arguments? - && - (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' + (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); - _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); + D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; done: @@ -39905,9 +33254,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _loop0_234: ',' double_starred_kvpair +// _loop0_126: ',' lambda_param static asdl_seq * -_loop0_234_rule(Parser *p) +_loop0_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39927,18 +33276,18 @@ _loop0_234_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' double_starred_kvpair + { // ',' lambda_param if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; - KeyValuePair* elem; + arg_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair + (elem = lambda_param_rule(p)) // lambda_param ) { _res = elem; @@ -39964,8 +33313,8 @@ _loop0_234_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_234[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c%s _loop0_126[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -39981,9 +33330,9 @@ _loop0_234_rule(Parser *p) return _seq; } -// _gather_233: double_starred_kvpair _loop0_234 +// _gather_127: lambda_param _loop0_126 static asdl_seq * -_gather_233_rule(Parser *p) +_gather_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39994,27 +33343,27 @@ _gather_233_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_234 + { // lambda_param _loop0_126 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_234")); - KeyValuePair* elem; + D(fprintf(stderr, "%*c> _gather_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_126")); + arg_ty elem; asdl_seq * seq; if ( - (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair + (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_234_rule(p)) // _loop0_234 + (seq = _loop0_126_rule(p)) // _loop0_126 ) { - D(fprintf(stderr, "%*c+ _gather_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_234")); + D(fprintf(stderr, "%*c+ _gather_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_126")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_233[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_234")); + D(fprintf(stderr, "%*c%s _gather_127[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_126")); } _res = NULL; done: @@ -40022,9 +33371,9 @@ _gather_233_rule(Parser *p) return _res; } -// _tmp_235: '}' | ',' +// _tmp_128: ',' | lambda_param_no_default static void * -_tmp_235_rule(Parser *p) +_tmp_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40035,43 +33384,43 @@ _tmp_235_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '}' + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } - { // ',' + { // lambda_param_no_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + arg_ty lambda_param_no_default_var; if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; done: @@ -40079,9 +33428,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: '}' | ',' +// _tmp_129: ':' | ',' (':' | '**') static void * -_tmp_236_rule(Parser *p) +_tmp_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40092,43 +33441,46 @@ _tmp_236_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '}' + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - { // ',' + { // ',' (':' | '**') if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; + void *_tmp_164_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (_tmp_164_var = _tmp_164_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_164_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; done: @@ -40136,9 +33488,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: '=' | '!' | ':' | '}' +// _tmp_130: lambda_param_no_default | ',' static void * -_tmp_237_rule(Parser *p) +_tmp_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40149,81 +33501,43 @@ _tmp_237_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); - } - { // '!' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' - ) - { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); - } - { // ':' + { // lambda_param_no_default if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + arg_ty lambda_param_no_default_var; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _tmp_130[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } - { // '}' + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _tmp_130[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; done: @@ -40231,9 +33545,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: '!' | ':' | '}' +// _tmp_131: bitwise_or ((',' bitwise_or))* ','? static void * -_tmp_238_rule(Parser *p) +_tmp_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40244,72 +33558,117 @@ _tmp_238_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '!' + { // bitwise_or ((',' bitwise_or))* ','? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + asdl_seq * _loop0_165_var; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty bitwise_or_var; if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + && + (_loop0_165_var = _loop0_165_rule(p)) // ((',' bitwise_or))* + && + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_165_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c%s _tmp_131[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); } - { // ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop0_132: ',' dotted_name +static asdl_seq * +_loop0_132_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); } - { // '}' + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' dotted_name if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _loop0_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + expr_ty elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = dotted_name_rule(p)) // dotted_name ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); - _res = _literal; - goto done; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _loop0_132[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } - _res = NULL; - done: + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); p->level--; - return _res; + return _seq; } -// _tmp_239: '!' NAME -static void * -_tmp_239_rule(Parser *p) +// _gather_133: dotted_name _loop0_132 +static asdl_seq * +_gather_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40318,29 +33677,29 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - { // '!' NAME + { // dotted_name _loop0_132 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - Token * _literal; - expr_ty name_var; + D(fprintf(stderr, "%*c> _gather_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_132")); + expr_ty elem; + asdl_seq * seq; if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' + (elem = dotted_name_rule(p)) // dotted_name && - (name_var = _PyPegen_name_token(p)) // NAME + (seq = _loop0_132_rule(p)) // _loop0_132 ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - _res = _PyPegen_dummy_name(p, _literal, name_var); + D(fprintf(stderr, "%*c+ _gather_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_132")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c%s _gather_133[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_132")); } _res = NULL; done: @@ -40348,9 +33707,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: ':' | '}' -static void * -_tmp_240_rule(Parser *p) +// _loop0_134: ',' (expression ['as' star_target]) +static asdl_seq * +_loop0_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40359,55 +33718,74 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; } - { // '}' + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // ',' (expression ['as' star_target]) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + void *elem; + while ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_166_rule(p)) // expression ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); - _res = _literal; - goto done; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _loop0_134[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } - _res = NULL; - done: + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); p->level--; - return _res; + return _seq; } -// _tmp_241: '!' NAME -static void * -_tmp_241_rule(Parser *p) +// _gather_135: (expression ['as' star_target]) _loop0_134 +static asdl_seq * +_gather_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40416,29 +33794,29 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - { // '!' NAME + { // (expression ['as' star_target]) _loop0_134 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - Token * _literal; - expr_ty name_var; + D(fprintf(stderr, "%*c> _gather_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_134")); + void *elem; + asdl_seq * seq; if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' + (elem = _tmp_166_rule(p)) // expression ['as' star_target] && - (name_var = _PyPegen_name_token(p)) // NAME + (seq = _loop0_134_rule(p)) // _loop0_134 ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - _res = _PyPegen_dummy_name(p, _literal, name_var); + D(fprintf(stderr, "%*c+ _gather_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_134")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c%s _gather_135[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_134")); } _res = NULL; done: @@ -40446,9 +33824,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _loop0_242: fstring_format_spec +// _loop0_136: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_242_rule(Parser *p) +_loop0_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40468,18 +33846,27 @@ _loop0_242_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // fstring_format_spec + { // ',' (expressions ['as' star_target]) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); - expr_ty fstring_format_spec_var; + D(fprintf(stderr, "%*c> _loop0_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + Token * _literal; + void *elem; while ( - (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (elem = _tmp_167_rule(p)) // expressions ['as' star_target] ) { - _res = fstring_format_spec_var; + _res = elem; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(_children); + p->level--; + return NULL; + } if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -40496,8 +33883,8 @@ _loop0_242_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_242[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c%s _loop0_136[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -40513,9 +33900,9 @@ _loop0_242_rule(Parser *p) return _seq; } -// _tmp_243: '!' NAME -static void * -_tmp_243_rule(Parser *p) +// _gather_137: (expressions ['as' star_target]) _loop0_136 +static asdl_seq * +_gather_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40524,29 +33911,29 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + asdl_seq * _res = NULL; int _mark = p->mark; - { // '!' NAME + { // (expressions ['as' star_target]) _loop0_136 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - Token * _literal; - expr_ty name_var; + D(fprintf(stderr, "%*c> _gather_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_136")); + void *elem; + asdl_seq * seq; if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' + (elem = _tmp_167_rule(p)) // expressions ['as' star_target] && - (name_var = _PyPegen_name_token(p)) // NAME + (seq = _loop0_136_rule(p)) // _loop0_136 ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); - _res = _PyPegen_dummy_name(p, _literal, name_var); + D(fprintf(stderr, "%*c+ _gather_137[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_136")); + _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c%s _gather_137[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_136")); } _res = NULL; done: @@ -40554,9 +33941,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _tmp_244: ':' | '}' +// _tmp_138: 'except' | 'finally' static void * -_tmp_244_rule(Parser *p) +_tmp_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40567,43 +33954,43 @@ _tmp_244_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ':' + { // 'except' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + Token * _keyword; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (_keyword = _PyPegen_expect_token(p, 663)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } - { // '}' + { // 'finally' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + Token * _keyword; if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' + (_keyword = _PyPegen_expect_token(p, 659)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; done: @@ -40611,9 +33998,9 @@ _tmp_244_rule(Parser *p) return _res; } -// _tmp_245: '+' | '-' | '*' | '/' | '%' | '//' | '@' -static void * -_tmp_245_rule(Parser *p) +// _loop0_139: block +static asdl_seq * +_loop0_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40622,140 +34009,97 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // '+' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - ) - { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); - } - { // '-' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 15)) // token='-' - ) - { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; } - { // '*' + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' + D(fprintf(stderr, "%*c> _loop0_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + asdl_stmt_seq* block_var; + while ( + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); - _res = _literal; - goto done; + _res = block_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c%s _loop0_139[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } - { // '/' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 17)) // token='/' - ) - { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; } - { // '%' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 24)) // token='%' - ) - { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_140: expression ['as' NAME] +static void * +_tmp_140_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); } - { // '//' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 47)) // token='//' - ) - { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); + if (p->error_indicator) { + p->level--; + return NULL; } - { // '@' + void * _res = NULL; + int _mark = p->mark; + { // expression ['as' NAME] if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty expression_var; if ( - (_literal = _PyPegen_expect_token(p, 49)) // token='@' + (expression_var = expression_rule(p)) // expression + && + (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c%s _tmp_140[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; done: @@ -40763,9 +34107,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '+' | '-' | '~' +// _tmp_141: NEWLINE | ':' static void * -_tmp_246_rule(Parser *p) +_tmp_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40776,62 +34120,43 @@ _tmp_246_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '+' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - ) - { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); - } - { // '-' + { // NEWLINE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + Token * newline_var; if ( - (_literal = _PyPegen_expect_token(p, 15)) // token='-' + (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c%s _tmp_141[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } - { // '~' + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 31)) // token='~' + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c%s _tmp_141[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; done: @@ -40839,9 +34164,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: star_targets '=' +// _tmp_142: positional_patterns ',' static void * -_tmp_247_rule(Parser *p) +_tmp_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40852,32 +34177,27 @@ _tmp_247_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // star_targets '=' + { // positional_patterns ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; - expr_ty z; + asdl_pattern_seq* positional_patterns_var; if ( - (z = star_targets_rule(p)) // star_targets + (positional_patterns_var = positional_patterns_rule(p)) // positional_patterns && - (_literal = _PyPegen_expect_token(p, 22)) // token='=' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c%s _tmp_142[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; done: @@ -40885,9 +34205,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: '.' | '...' +// _tmp_143: '}' | ',' static void * -_tmp_248_rule(Parser *p) +_tmp_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40898,43 +34218,43 @@ _tmp_248_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '.' + { // '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 23)) // token='.' + (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c%s _tmp_143[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } - { // '...' + { // ',' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 52)) // token='...' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c%s _tmp_143[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; done: @@ -40942,9 +34262,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: '.' | '...' +// _tmp_144: '=' | '!' | ':' | '}' static void * -_tmp_249_rule(Parser *p) +_tmp_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40955,138 +34275,81 @@ _tmp_249_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '.' + { // '=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 23)) // token='.' + (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } - { // '...' + { // '!' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( - (_literal = _PyPegen_expect_token(p, 52)) // token='...' + (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_250: '@' named_expression NEWLINE -static void * -_tmp_250_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } - void * _res = NULL; - int _mark = p->mark; - { // '@' named_expression NEWLINE + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; - expr_ty f; - Token * newline_var; if ( - (_literal = _PyPegen_expect_token(p, 49)) // token='@' - && - (f = named_expression_rule(p)) // named_expression - && - (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); - _res = f; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_251: ',' expression -static void * -_tmp_251_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - void * _res = NULL; - int _mark = p->mark; - { // ',' expression + { // '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; - expr_ty c; if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (c = expression_rule(p)) // expression + (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; done: @@ -41094,9 +34357,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: ',' star_expression +// _tmp_145: '!' | ':' | '}' static void * -_tmp_252_rule(Parser *p) +_tmp_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41107,78 +34370,62 @@ _tmp_252_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ',' star_expression + { // '!' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; - expr_ty c; if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (c = star_expression_rule(p)) // star_expression + (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_253: 'or' conjunction -static void * -_tmp_253_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); + D(fprintf(stderr, "%*c%s _tmp_145[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } - if (p->error_indicator) { - p->level--; - return NULL; + { // ':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + ) + { + D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_145[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - void * _res = NULL; - int _mark = p->mark; - { // 'or' conjunction + { // '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); - Token * _keyword; - expr_ty c; + D(fprintf(stderr, "%*c> _tmp_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 581)) // token='or' - && - (c = conjunction_rule(p)) // conjunction + (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c%s _tmp_145[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; done: @@ -41186,9 +34433,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'and' inversion +// _tmp_146: '!' NAME static void * -_tmp_254_rule(Parser *p) +_tmp_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41199,32 +34446,27 @@ _tmp_254_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'and' inversion + { // '!' NAME if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); - Token * _keyword; - expr_ty c; + D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + Token * _literal; + expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 582)) // token='and' + (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - (c = inversion_rule(p)) // inversion + (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; done: @@ -41232,9 +34474,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: slice | starred_expression +// _tmp_147: ':' | '}' static void * -_tmp_255_rule(Parser *p) +_tmp_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41245,43 +34487,43 @@ _tmp_255_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // slice + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); - expr_ty slice_var; + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; if ( - (slice_var = slice_rule(p)) // slice + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); - _res = slice_var; + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - { // starred_expression + { // '}' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); - expr_ty starred_expression_var; + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + Token * _literal; if ( - (starred_expression_var = starred_expression_rule(p)) // starred_expression + (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); - _res = starred_expression_var; + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; done: @@ -41289,9 +34531,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _tmp_256: fstring | string +// _tmp_148: '+' | '-' | '*' | '/' | '%' | '//' | '@' static void * -_tmp_256_rule(Parser *p) +_tmp_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41302,135 +34544,138 @@ _tmp_256_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // fstring + { // '+' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); - expr_ty fstring_var; + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + Token * _literal; if ( - (fstring_var = fstring_rule(p)) // fstring + (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); - _res = fstring_var; + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } - { // string + { // '-' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); - expr_ty string_var; + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; if ( - (string_var = string_rule(p)) // string + (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); - _res = string_var; + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_257: 'if' disjunction -static void * -_tmp_257_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } - if (p->error_indicator) { - p->level--; - return NULL; + { // '*' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + ) + { + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } - void * _res = NULL; - int _mark = p->mark; - { // 'if' disjunction + { // '/' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); - Token * _keyword; - expr_ty z; + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='if' - && - (z = disjunction_rule(p)) // disjunction + (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_258: 'if' disjunction -static void * -_tmp_258_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); + { // '%' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 24)) // token='%' + ) + { + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } - if (p->error_indicator) { - p->level--; - return NULL; + { // '//' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 47)) // token='//' + ) + { + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } - void * _res = NULL; - int _mark = p->mark; - { // 'if' disjunction + { // '@' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); - Token * _keyword; - expr_ty z; + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 668)) // token='if' - && - (z = disjunction_rule(p)) // disjunction + (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; done: @@ -41438,9 +34683,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_149: '+' | '-' | '~' static void * -_tmp_259_rule(Parser *p) +_tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41451,45 +34696,62 @@ _tmp_259_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // starred_expression + { // '+' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); - expr_ty starred_expression_var; + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + Token * _literal; if ( - (starred_expression_var = starred_expression_rule(p)) // starred_expression + (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); - _res = starred_expression_var; + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } - { // (assignment_expression | expression !':=') !'=' + { // '-' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_274_var; + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; if ( - (_tmp_274_var = _tmp_274_rule(p)) // assignment_expression | expression !':=' - && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' + (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_274_var; + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + } + { // '~' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 31)) // token='~' + ) + { + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; done: @@ -41497,9 +34759,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: ',' star_target +// _tmp_150: star_targets '=' static void * -_tmp_260_rule(Parser *p) +_tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41510,22 +34772,22 @@ _tmp_260_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ',' star_target + { // star_targets '=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; - expr_ty c; + expr_ty z; if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + (z = star_targets_rule(p)) // star_targets && - (c = star_target_rule(p)) // star_target + (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); - _res = c; + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -41534,8 +34796,8 @@ _tmp_260_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; done: @@ -41543,9 +34805,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: ',' star_target +// _tmp_151: '.' | '...' static void * -_tmp_261_rule(Parser *p) +_tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41556,32 +34818,43 @@ _tmp_261_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ',' star_target + { // '.' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; - expr_ty c; if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (c = star_target_rule(p)) // star_target + (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + } + { // '...' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 52)) // token='...' + ) + { + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; done: @@ -41589,10 +34862,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: -// | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs +// _tmp_152: '@' named_expression NEWLINE static void * -_tmp_262_rule(Parser *p) +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41603,30 +34875,35 @@ _tmp_262_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + { // '@' named_expression NEWLINE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_275_var; + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; - asdl_seq* kwargs_var; + expr_ty f; + Token * newline_var; if ( - (_gather_275_var = _gather_275_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_literal = _PyPegen_expect_token(p, 49)) // token='@' && - (_literal = _PyPegen_expect_token(p, 12)) // token=',' + (f = named_expression_rule(p)) // named_expression && - (kwargs_var = kwargs_rule(p)) // kwargs + (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_275_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + _res = f; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; done: @@ -41634,9 +34911,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: starred_expression !'=' +// _tmp_153: ',' star_expression static void * -_tmp_263_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41647,26 +34924,32 @@ _tmp_263_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // starred_expression !'=' + { // ',' star_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); - expr_ty starred_expression_var; + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + Token * _literal; + expr_ty c; if ( - (starred_expression_var = starred_expression_rule(p)) // starred_expression + (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' + (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); - _res = starred_expression_var; + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; done: @@ -41674,9 +34957,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: star_targets '=' +// _tmp_154: 'or' conjunction static void * -_tmp_264_rule(Parser *p) +_tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41687,27 +34970,32 @@ _tmp_264_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // star_targets '=' + { // 'or' conjunction if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - Token * _literal; - expr_ty star_targets_var; + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + Token * _keyword; + expr_ty c; if ( - (star_targets_var = star_targets_rule(p)) // star_targets + (_keyword = _PyPegen_expect_token(p, 581)) // token='or' && - (_literal = _PyPegen_expect_token(p, 22)) // token='=' + (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - _res = _PyPegen_dummy_name(p, star_targets_var, _literal); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; done: @@ -41715,9 +35003,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: star_targets '=' +// _tmp_155: 'and' inversion static void * -_tmp_265_rule(Parser *p) +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41728,27 +35016,32 @@ _tmp_265_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // star_targets '=' + { // 'and' inversion if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - Token * _literal; - expr_ty star_targets_var; + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + Token * _keyword; + expr_ty c; if ( - (star_targets_var = star_targets_rule(p)) // star_targets + (_keyword = _PyPegen_expect_token(p, 582)) // token='and' && - (_literal = _PyPegen_expect_token(p, 22)) // token='=' + (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - _res = _PyPegen_dummy_name(p, star_targets_var, _literal); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; done: @@ -41756,9 +35049,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ')' | '**' +// _tmp_156: slice | starred_expression static void * -_tmp_266_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41769,43 +35062,43 @@ _tmp_266_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ')' + { // slice if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + expr_ty slice_var; if ( - (_literal = _PyPegen_expect_token(p, 8)) // token=')' + (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } - { // '**' + { // starred_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + expr_ty starred_expression_var; if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' + (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; done: @@ -41813,9 +35106,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: ':' | '**' +// _tmp_157: fstring | string static void * -_tmp_267_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41826,43 +35119,43 @@ _tmp_267_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // ':' + { // fstring if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + expr_ty fstring_var; if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' + (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } - { // '**' + { // string if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); - Token * _literal; + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + expr_ty string_var; if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' + (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); - _res = _literal; + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; done: @@ -41870,9 +35163,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _loop0_268: (',' bitwise_or) -static asdl_seq * -_loop0_268_rule(Parser *p) +// _tmp_158: 'if' disjunction +static void * +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41881,65 +35174,44 @@ _loop0_268_rule(Parser *p) p->level--; return NULL; } - void *_res = NULL; + void * _res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (',' bitwise_or) + { // 'if' disjunction if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_277_var; - while ( - (_tmp_277_var = _tmp_277_rule(p)) // ',' bitwise_or + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + Token * _keyword; + expr_ty z; + if ( + (_keyword = _PyPegen_expect_token(p, 668)) // token='if' + && + (z = disjunction_rule(p)) // disjunction ) { - _res = _tmp_277_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + _res = z; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; } - _children[_n++] = _res; - _mark = p->mark; + goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_268[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); + _res = NULL; + done: p->level--; - return _seq; + return _res; } -// _tmp_269: expression ['as' star_target] +// _tmp_159: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_269_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41950,28 +35222,45 @@ _tmp_269_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // expression ['as' star_target] + { // starred_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty expression_var; + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + expr_ty starred_expression_var; if ( - (expression_var = expression_rule(p)) // expression + (starred_expression_var = starred_expression_rule(p)) // starred_expression + ) + { + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + _res = starred_expression_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); + } + { // (assignment_expression | expression !':=') !'=' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_85_var; + if ( + (_tmp_85_var = _tmp_85_rule(p)) // assignment_expression | expression !':=' && - (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); - _res = _PyPegen_dummy_name(p, expression_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_85_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; done: @@ -41979,9 +35268,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: expressions ['as' star_target] +// _tmp_160: ',' star_target static void * -_tmp_270_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41992,28 +35281,32 @@ _tmp_270_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // expressions ['as' star_target] + { // ',' star_target if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty expressions_var; + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + Token * _literal; + expr_ty c; if ( - (expressions_var = expressions_rule(p)) // expressions + (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' star_target] + (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); - _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; done: @@ -42021,9 +35314,10 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: expression ['as' star_target] +// _tmp_161: +// | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_271_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42034,28 +35328,30 @@ _tmp_271_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // expression ['as' star_target] + { // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty expression_var; + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_87_var; + Token * _literal; + asdl_seq* kwargs_var; if ( - (expression_var = expression_rule(p)) // expression + (_gather_87_var = _gather_87_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + && + (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_280_rule(p), !p->error_indicator) // ['as' star_target] + (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); - _res = _PyPegen_dummy_name(p, expression_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_87_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; done: @@ -42063,9 +35359,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: expressions ['as' star_target] +// _tmp_162: starred_expression !'=' static void * -_tmp_272_rule(Parser *p) +_tmp_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42076,28 +35372,26 @@ _tmp_272_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // expressions ['as' star_target] + { // starred_expression !'=' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty expressions_var; + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + expr_ty starred_expression_var; if ( - (expressions_var = expressions_rule(p)) // expressions + (starred_expression_var = starred_expression_rule(p)) // starred_expression && - (_opt_var = _tmp_281_rule(p), !p->error_indicator) // ['as' star_target] + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); - _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; done: @@ -42105,9 +35399,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _tmp_273: 'as' NAME +// _tmp_163: ')' | '**' static void * -_tmp_273_rule(Parser *p) +_tmp_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42118,27 +35412,43 @@ _tmp_273_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' NAME + { // ')' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - Token * _keyword; - expr_ty name_var; + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + Token * _literal; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' - && - (name_var = _PyPegen_name_token(p)) // NAME + (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); - _res = _PyPegen_dummy_name(p, _keyword, name_var); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); + } + { // '**' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 35)) // token='**' + ) + { + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; done: @@ -42146,9 +35456,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _tmp_274: assignment_expression | expression !':=' +// _tmp_164: ':' | '**' static void * -_tmp_274_rule(Parser *p) +_tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42159,45 +35469,43 @@ _tmp_274_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // assignment_expression + { // ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); - expr_ty assignment_expression_var; + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; if ( - (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression + (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); - _res = assignment_expression_var; + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } - { // expression !':=' + { // '**' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); - expr_ty expression_var; + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + Token * _literal; if ( - (expression_var = expression_rule(p)) // expression - && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' + (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); - _res = expression_var; + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_274[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; done: @@ -42205,9 +35513,9 @@ _tmp_274_rule(Parser *p) return _res; } -// _loop0_276: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_165: (',' bitwise_or) static asdl_seq * -_loop0_276_rule(Parser *p) +_loop0_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42227,27 +35535,18 @@ _loop0_276_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') + { // (',' bitwise_or) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); - Token * _literal; - void *elem; + D(fprintf(stderr, "%*c> _loop0_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_168_var; while ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (elem = _tmp_282_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (_tmp_168_var = _tmp_168_rule(p)) // ',' bitwise_or ) { - _res = elem; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(_children); - p->level--; - return NULL; - } + _res = _tmp_168_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -42264,8 +35563,8 @@ _loop0_276_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_276[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c%s _loop0_165[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { @@ -42281,92 +35580,9 @@ _loop0_276_rule(Parser *p) return _seq; } -// _gather_275: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_276 -static asdl_seq * -_gather_275_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_seq * _res = NULL; - int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_276 - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _gather_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_276")); - void *elem; - asdl_seq * seq; - if ( - (elem = _tmp_282_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' - && - (seq = _loop0_276_rule(p)) // _loop0_276 - ) - { - D(fprintf(stderr, "%*c+ _gather_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_276")); - _res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_275[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_276")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_277: ',' bitwise_or -static void * -_tmp_277_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' bitwise_or - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); - Token * _literal; - expr_ty bitwise_or_var; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or - ) - { - D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); - _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_278: 'as' star_target +// _tmp_166: expression ['as' star_target] static void * -_tmp_278_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42377,27 +35593,28 @@ _tmp_278_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' star_target + { // expression ['as' star_target] if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); - Token * _keyword; - expr_ty star_target_var; + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' + (expression_var = expression_rule(p)) // expression && - (star_target_var = star_target_rule(p)) // star_target + (_opt_var = _tmp_169_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); - _res = _PyPegen_dummy_name(p, _keyword, star_target_var); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; done: @@ -42405,9 +35622,9 @@ _tmp_278_rule(Parser *p) return _res; } -// _tmp_279: 'as' star_target +// _tmp_167: expressions ['as' star_target] static void * -_tmp_279_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42418,27 +35635,28 @@ _tmp_279_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' star_target + { // expressions ['as' star_target] if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); - Token * _keyword; - expr_ty star_target_var; + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty expressions_var; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' + (expressions_var = expressions_rule(p)) // expressions && - (star_target_var = star_target_rule(p)) // star_target + (_opt_var = _tmp_169_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); - _res = _PyPegen_dummy_name(p, _keyword, star_target_var); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; done: @@ -42446,9 +35664,9 @@ _tmp_279_rule(Parser *p) return _res; } -// _tmp_280: 'as' star_target +// _tmp_168: ',' bitwise_or static void * -_tmp_280_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42459,27 +35677,27 @@ _tmp_280_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // 'as' star_target + { // ',' bitwise_or if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); - Token * _keyword; - expr_ty star_target_var; + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + Token * _literal; + expr_ty bitwise_or_var; if ( - (_keyword = _PyPegen_expect_token(p, 666)) // token='as' + (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (star_target_var = star_target_rule(p)) // star_target + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); - _res = _PyPegen_dummy_name(p, _keyword, star_target_var); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; done: @@ -42487,9 +35705,9 @@ _tmp_280_rule(Parser *p) return _res; } -// _tmp_281: 'as' star_target +// _tmp_169: 'as' star_target static void * -_tmp_281_rule(Parser *p) +_tmp_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42505,7 +35723,7 @@ _tmp_281_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42514,12 +35732,12 @@ _tmp_281_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42528,124 +35746,6 @@ _tmp_281_rule(Parser *p) return _res; } -// _tmp_282: starred_expression | (assignment_expression | expression !':=') !'=' -static void * -_tmp_282_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // starred_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_282[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); - expr_ty starred_expression_var; - if ( - (starred_expression_var = starred_expression_rule(p)) // starred_expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_282[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); - _res = starred_expression_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_282[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); - } - { // (assignment_expression | expression !':=') !'=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_282[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_283_var; - if ( - (_tmp_283_var = _tmp_283_rule(p)) // assignment_expression | expression !':=' - && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_282[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_283_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_282[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_283: assignment_expression | expression !':=' -static void * -_tmp_283_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // assignment_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_283[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); - expr_ty assignment_expression_var; - if ( - (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_283[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); - _res = assignment_expression_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_283[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); - } - { // expression !':=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_283[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); - expr_ty expression_var; - if ( - (expression_var = expression_rule(p)) // expression - && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_283[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); - _res = expression_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_283[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); - } - _res = NULL; - done: - p->level--; - return _res; -} - void * _PyPegen_parse(Parser *p) { diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e94a4923228d0f..6146f69912bfa3 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -276,7 +276,7 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) assert(p->tok->fp_interactive); // We can reach this point if the tokenizer buffers for interactive source have not been // initialized because we failed to decode the original source with the given locale. - return PyUnicode_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; @@ -359,7 +359,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, error_line = get_error_line_from_tokenizer_buffers(p, lineno); } else { - error_line = PyUnicode_FromStringAndSize("", 0); + error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } if (!error_line) { goto error; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 10ee6b7be23e21..d15dd519dbf6af 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -311,14 +311,36 @@ static int test_pre_initialization_api(void) _Py_EMBED_PREINIT_CHECK("Checking Py_SetProgramName\n"); Py_SetProgramName(program); + _Py_EMBED_PREINIT_CHECK("Checking !Py_IsInitialized pre-initialization\n"); + if (Py_IsInitialized()) { + fprintf(stderr, "Fatal error: initialized before initialization!\n"); + return 1; + } + _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); Py_Initialize(); + + _Py_EMBED_PREINIT_CHECK("Checking Py_IsInitialized post-initialization\n"); + if (!Py_IsInitialized()) { + fprintf(stderr, "Fatal error: not initialized after initialization!\n"); + return 1; + } + _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); - PyRun_SimpleString("import sys; " - "print('sys.executable:', sys.executable)"); + PyRun_SimpleString( + "import sys; " + "print('sys.executable:', sys.executable); " + "sys.stdout.flush(); " + ); _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n"); Py_Finalize(); + _Py_EMBED_PREINIT_CHECK("Checking !Py_IsInitialized post-finalization\n"); + if (Py_IsInitialized()) { + fprintf(stderr, "Fatal error: still initialized after finalization!\n"); + return 1; + } + _Py_EMBED_PREINIT_CHECK("Freeing memory allocated by Py_DecodeLocale\n"); PyMem_RawFree(program); return 0; @@ -364,12 +386,15 @@ static int test_pre_initialization_sys_options(void) _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); _testembed_Py_InitializeFromConfig(); _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); - PyRun_SimpleString("import sys; " - "print('sys.warnoptions:', sys.warnoptions); " - "print('sys._xoptions:', sys._xoptions); " - "warnings = sys.modules['warnings']; " - "latest_filters = [f[0] for f in warnings.filters[:3]]; " - "print('warnings.filters[:3]:', latest_filters)"); + PyRun_SimpleString( + "import sys; " + "print('sys.warnoptions:', sys.warnoptions); " + "print('sys._xoptions:', sys._xoptions); " + "warnings = sys.modules['warnings']; " + "latest_filters = [f[0] for f in warnings.filters[:3]]; " + "print('warnings.filters[:3]:', latest_filters); " + "sys.stdout.flush(); " + ); _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n"); Py_Finalize(); @@ -810,6 +835,7 @@ static void set_most_env_vars(void) #ifdef Py_STATS putenv("PYTHONSTATS=1"); #endif + putenv("PYTHONPERFSUPPORT=1"); } @@ -1844,6 +1870,10 @@ static int test_initconfig_api(void) goto error; } + if (PyInitConfig_SetInt(config, "perf_profiling", 2) < 0) { + goto error; + } + // Set a UTF-8 string (program_name) if (PyInitConfig_SetStr(config, "program_name", PROGRAM_NAME_UTF8) < 0) { goto error; @@ -1866,16 +1896,19 @@ static int test_initconfig_api(void) goto error; } PyInitConfig_Free(config); + PyInitConfig_Free(NULL); dump_config(); Py_Finalize(); return 0; - const char *err_msg; error: - (void)PyInitConfig_GetError(config, &err_msg); - printf("Python init failed: %s\n", err_msg); - exit(1); + { + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("Python init failed: %s\n", err_msg); + exit(1); + } } @@ -2019,11 +2052,13 @@ static int test_initconfig_module(void) Py_Finalize(); return 0; - const char *err_msg; error: - (void)PyInitConfig_GetError(config, &err_msg); - printf("Python init failed: %s\n", err_msg); - exit(1); + { + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("Python init failed: %s\n", err_msg); + exit(1); + } } diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 860447ef9ed702..38d74b48d232f8 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -19,7 +19,7 @@ struct validator { }; // Forward declaration -static int init_types(struct ast_state *state); +static int init_types(void *arg); static struct ast_state* get_ast_state(void) @@ -281,8 +281,6 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->vararg); Py_CLEAR(state->withitem_type); - Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); - state->finalized = 1; state->once = (_PyOnceFlag){0}; } @@ -5044,8 +5042,9 @@ typedef struct { } AST_object; static void -ast_dealloc(AST_object *self) +ast_dealloc(PyObject *op) { + AST_object *self = (AST_object*)op; /* bpo-31095: UnTrack is needed before calling any callbacks */ PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); @@ -5057,16 +5056,18 @@ ast_dealloc(AST_object *self) } static int -ast_traverse(AST_object *self, visitproc visit, void *arg) +ast_traverse(PyObject *op, visitproc visit, void *arg) { + AST_object *self = (AST_object*)op; Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -ast_clear(AST_object *self) +ast_clear(PyObject *op) { + AST_object *self = (AST_object*)op; Py_CLEAR(self->dict); return 0; } @@ -5806,7 +5807,6 @@ ast_repr_max_depth(AST_object *self, int depth) if (!value_repr) { Py_DECREF(name); - Py_DECREF(value); goto error; } @@ -5852,9 +5852,9 @@ ast_repr_max_depth(AST_object *self, int depth) } static PyObject * -ast_repr(AST_object *self) +ast_repr(PyObject *self) { - return ast_repr_max_depth(self, 3); + return ast_repr_max_depth((AST_object*)self, 3); } static PyType_Slot AST_type_slots[] = { @@ -6047,8 +6047,9 @@ static int add_ast_fields(struct ast_state *state) static int -init_types(struct ast_state *state) +init_types(void *arg) { + struct ast_state *state = arg; if (init_identifiers(state) < 0) { return -1; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 34b4445be27f62..50ce83d18f6e73 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -263,7 +263,7 @@ tokenizeriter_next(tokenizeriterobject *it) } PyObject *str = NULL; if (token.start == NULL || token.end == NULL) { - str = PyUnicode_FromString(""); + str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { str = PyUnicode_FromStringAndSize(token.start, token.end - token.start); @@ -281,7 +281,7 @@ tokenizeriter_next(tokenizeriterobject *it) PyObject* line = NULL; int line_changed = 1; if (it->tok->tok_extra_tokens && is_trailing_token) { - line = PyUnicode_FromString(""); + line = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { Py_ssize_t size = it->tok->inp - line_start; if (size >= 1 && it->tok->implicit_newline) { @@ -326,7 +326,7 @@ tokenizeriter_next(tokenizeriterobject *it) else if (type == NL) { if (it->tok->implicit_newline) { Py_DECREF(str); - str = PyUnicode_FromString(""); + str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f5b04757e08bf3..01e208b88eca8b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -169,11 +169,10 @@ safe_multiply(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); - uint64_t wbits = _PyLong_NumBits(w); - if (vbits == (uint64_t)-1 || wbits == (uint64_t)-1) { - return NULL; - } + int64_t vbits = _PyLong_NumBits(v); + int64_t wbits = _PyLong_NumBits(w); + assert(vbits >= 0); + assert(wbits >= 0); if (vbits + wbits > MAX_INT_SIZE) { return NULL; } @@ -215,12 +214,13 @@ safe_power(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && _PyLong_IsPositive((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (vbits > MAX_INT_SIZE / wbits) { + if ((uint64_t)vbits > MAX_INT_SIZE / wbits) { return NULL; } } @@ -234,12 +234,13 @@ safe_lshift(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (wbits > MAX_INT_SIZE || vbits > MAX_INT_SIZE - wbits) { + if (wbits > MAX_INT_SIZE || (uint64_t)vbits > MAX_INT_SIZE - wbits) { return NULL; } } diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 86f7a582b981a3..8017cfc7fcf268 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -2,7 +2,6 @@ #include "pycore_ast.h" // expr_ty #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _Py_ID() -#include // DBL_MAX_10_EXP #include /* This limited unparser is used to convert annotations back to strings @@ -13,10 +12,6 @@ _Py_DECLARE_STR(dbl_open_br, "{{"); _Py_DECLARE_STR(dbl_close_br, "}}"); -/* We would statically initialize this if doing so were simple enough. */ -#define _str_replace_inf(interp) \ - _Py_INTERP_CACHED_OBJECT(interp, str_replace_inf) - /* Forward declarations for recursion via helper functions. */ static PyObject * expr_as_unicode(expr_ty e, int level); @@ -78,13 +73,13 @@ append_repr(_PyUnicodeWriter *writer, PyObject *obj) } if ((PyFloat_CheckExact(obj) && isinf(PyFloat_AS_DOUBLE(obj))) || - PyComplex_CheckExact(obj)) + PyComplex_CheckExact(obj)) { - PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_DECLARE_STR(str_replace_inf, "1e309"); // evaluates to inf PyObject *new_repr = PyUnicode_Replace( repr, &_Py_ID(inf), - _str_replace_inf(interp), + &_Py_STR(str_replace_inf), -1 ); Py_DECREF(repr); @@ -918,20 +913,6 @@ append_ast_expr(_PyUnicodeWriter *writer, expr_ty e, int level) return -1; } -static int -maybe_init_static_strings(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_str_replace_inf(interp) == NULL) { - PyObject *tmp = PyUnicode_FromFormat("1e%d", 1 + DBL_MAX_10_EXP); - if (tmp == NULL) { - return -1; - } - _str_replace_inf(interp) = tmp; - } - return 0; -} - static PyObject * expr_as_unicode(expr_ty e, int level) { @@ -939,9 +920,7 @@ expr_as_unicode(expr_ty e, int level) _PyUnicodeWriter_Init(&writer); writer.min_length = 256; writer.overallocate = 1; - if (-1 == maybe_init_static_strings() || - -1 == append_ast_expr(&writer, e, level)) - { + if (-1 == append_ast_expr(&writer, e, level)) { _PyUnicodeWriter_Dealloc(&writer); return NULL; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 846404e28bb18f..e6525657cabc2b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -149,7 +149,7 @@ dummy_func( op(_CHECK_PERIODIC, (--)) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { int err = _Py_HandlePending(tstate); ERROR_IF(err != 0, error); @@ -207,10 +207,8 @@ dummy_func( } op(_MONITOR_RESUME, (--)) { - _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ @@ -263,6 +261,7 @@ dummy_func( replicate(8) inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); + DEAD(value); } pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = { @@ -273,6 +272,7 @@ dummy_func( uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); + DEAD(value1); value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } @@ -280,7 +280,9 @@ dummy_func( uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); + DEAD(value1); SETLOCAL(oparg2, value2); + DEAD(value2); } pure inst(POP_TOP, (value --)) { @@ -305,12 +307,14 @@ dummy_func( DECREF_INPUTS(); } - pure inst(END_SEND, (receiver, value -- value)) { + pure inst(END_SEND, (receiver, value -- val)) { (void)receiver; + val = value; + DEAD(value); PyStackRef_CLOSE(receiver); } - tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- val)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -318,6 +322,8 @@ dummy_func( ERROR_NO_POP(); } } + val = value; + DEAD(value); PyStackRef_CLOSE(receiver); } @@ -332,6 +338,7 @@ dummy_func( assert(PyStackRef_BoolCheck(value)); res = PyStackRef_Is(value, PyStackRef_False) ? PyStackRef_True : PyStackRef_False; + DEAD(value); } family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = { @@ -374,7 +381,8 @@ dummy_func( EXIT_IF(!PyLong_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortalLoose(value_o)); + assert(_Py_IsImmortal(value_o)); + DEAD(value); res = PyStackRef_False; } else { @@ -394,6 +402,7 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: EXIT_IF(!PyStackRef_Is(value, PyStackRef_None)); + DEAD(value); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; } @@ -403,7 +412,8 @@ dummy_func( EXIT_IF(!PyUnicode_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortalLoose(value_o)); + assert(_Py_IsImmortal(value_o)); + DEAD(value); res = PyStackRef_False; } else { @@ -464,8 +474,9 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -476,8 +487,9 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -488,8 +500,9 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -526,8 +539,9 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + INPUTS_DEAD(); + ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -539,8 +553,9 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + INPUTS_DEAD(); + ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -552,8 +567,9 @@ dummy_func( double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + INPUTS_DEAD(); + ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -578,8 +594,9 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -594,7 +611,9 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { + #ifndef NDEBUG PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + #endif PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); int next_oparg; @@ -619,11 +638,13 @@ dummy_func( * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - _Py_DECREF_NO_DEALLOC(left_o); + PyStackRef_CLOSE(left); + DEAD(left); PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + DEAD(right); ERROR_IF(PyStackRef_IsNull(*target_local), error); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -645,6 +666,7 @@ dummy_func( }; specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { + assert(frame->stackpointer == NULL); #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -735,7 +757,8 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -754,7 +777,8 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -774,7 +798,8 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -808,17 +833,16 @@ dummy_func( assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); } op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _PyInterpreterFrame* )) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); - SYNC_SP(); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; + INPUTS_DEAD(); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } @@ -830,8 +854,9 @@ dummy_func( _PUSH_FRAME; inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) { - ERROR_IF(_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)) < 0, error); + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + ERROR_IF(err < 0, error); } inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { @@ -885,7 +910,8 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + DEAD(sub_st); PyStackRef_CLOSE(list_st); } @@ -929,25 +955,14 @@ dummy_func( } tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = PyStackRef_AsPyObjectSteal(args[1]); - _Py_FALLTHROUGH; - case 1: - exc = PyStackRef_AsPyObjectSteal(args[0]); - _Py_FALLTHROUGH; - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + int err = do_raise(tstate, exc, cause); + if (err) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } ERROR_IF(true, error); } @@ -970,17 +985,18 @@ dummy_func( #if TIER_ONE assert(frame != &entry_frame); #endif - SYNC_SP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef temp = retval; + DEAD(retval); + SAVE_STACK(); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + RELOAD_STACK(); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } @@ -988,7 +1004,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - if (err) ERROR_NO_POP(); + ERROR_IF(err, error); } macro(INSTRUMENTED_RETURN_VALUE) = @@ -1112,7 +1128,8 @@ dummy_func( JUMPBY(oparg); } else { - ERROR_NO_POP(); + DECREF_INPUTS(); + ERROR_IF(true, error); } } PyStackRef_CLOSE(v); @@ -1128,6 +1145,7 @@ dummy_func( STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); + DEAD(v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -1154,8 +1172,9 @@ dummy_func( assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - SYNC_SP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef temp = retval; + DEAD(retval); + SAVE_STACK(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1172,19 +1191,19 @@ dummy_func( _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif + RELOAD_STACK(); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - LOAD_SP(); - value = retval; + value = temp; LLTRACE_RESUME_FRAME(); } tier1 op(_YIELD_VALUE_EVENT, (val -- val)) { - SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - LOAD_SP(); - if (err) ERROR_NO_POP(); + if (err) { + ERROR_NO_POP(); + } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; DISPATCH(); @@ -1203,7 +1222,7 @@ dummy_func( } tier1 inst(RERAISE, (values[oparg], exc_st -- values[oparg])) { - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { @@ -1214,11 +1233,11 @@ dummy_func( } else { _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + Py_DECREF(exc); ERROR_NO_POP(); } } assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); goto exception_unwind; @@ -1228,7 +1247,8 @@ dummy_func( PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + if (matches) { DECREF_INPUTS(); } else { @@ -1246,9 +1266,9 @@ dummy_func( int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { + none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); - none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); @@ -1259,21 +1279,22 @@ dummy_func( inst(LOAD_COMMON_CONSTANT, ( -- value)) { // Keep in sync with _common_constants in opcode.py - switch(oparg) { - case CONSTANT_ASSERTIONERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); - break; - case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); - break; - default: - Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); } inst(LOAD_BUILD_CLASS, ( -- bc)) { PyObject *bc_o; - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0, error); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + ERROR_IF(err < 0, error); if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -1292,10 +1313,12 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(true, error); } - if (PyDict_CheckExact(ns)) + if (PyDict_CheckExact(ns)) { err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - else + } + else { err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + } DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1461,9 +1484,8 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - if (err < 0) { - ERROR_NO_POP(); - } + DECREF_INPUTS(); + ERROR_IF(err < 0, error); if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -1484,10 +1506,12 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0, error); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + ERROR_IF(err < 0, error); if (v_o == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + ERROR_IF(err < 0, error); if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, @@ -1497,7 +1521,6 @@ dummy_func( } } } - DECREF_INPUTS(); v = PyStackRef_FromPyObjectSteal(v_o); } @@ -1548,17 +1571,29 @@ dummy_func( assert(DK_IS_UNICODE(dict->ma_keys)); } - op(_GUARD_BUILTINS_VERSION, (version/1 --)) { + op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version / 1 -- globals_keys: PyDictKeysObject *)) + { + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); + globals_keys = dict->ma_keys; + assert(DK_IS_UNICODE(globals_keys)); + } + + op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version / 1 -- builtins_keys: PyDictKeysObject *)) + { PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict)); DEOPT_IF(dict->ma_keys->dk_version != version); - assert(DK_IS_UNICODE(dict->ma_keys)); + builtins_keys = dict->ma_keys; + assert(DK_IS_UNICODE(builtins_keys)); } - op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res, null if (oparg & 1))) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = entries[index].me_value; + DEAD(globals_keys); + SYNC_SP(); DEOPT_IF(res_o == NULL); Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); @@ -1566,10 +1601,11 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res, null if (oparg & 1))) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = entries[index].me_value; + DEAD(builtins_keys); + SYNC_SP(); DEOPT_IF(res_o == NULL); Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); @@ -1579,15 +1615,15 @@ dummy_func( macro(LOAD_GLOBAL_MODULE) = unused/1 + // Skip over the counter - _GUARD_GLOBALS_VERSION + + _GUARD_GLOBALS_VERSION_PUSH_KEYS + unused/1 + // Skip over the builtins version - _LOAD_GLOBAL_MODULE; + _LOAD_GLOBAL_MODULE_FROM_KEYS; macro(LOAD_GLOBAL_BUILTIN) = unused/1 + // Skip over the counter _GUARD_GLOBALS_VERSION + - _GUARD_BUILTINS_VERSION + - _LOAD_GLOBAL_BUILTINS; + _GUARD_BUILTINS_VERSION_PUSH_KEYS + + _LOAD_GLOBAL_BUILTINS_FROM_KEYS; inst(DELETE_FAST, (--)) { _PyStackRef v = GETLOCAL(oparg); @@ -1666,8 +1702,9 @@ dummy_func( inst(COPY_FREE_VARS, (--)) { /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { @@ -1691,12 +1728,14 @@ dummy_func( inst(BUILD_TUPLE, (values[oparg] -- tup)) { PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + INPUTS_DEAD(); ERROR_IF(tup_o == NULL, error); tup = PyStackRef_FromPyObjectSteal(tup_o); } inst(BUILD_LIST, (values[oparg] -- list)) { PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); + INPUTS_DEAD(); ERROR_IF(list_o == NULL, error); list = PyStackRef_FromPyObjectSteal(list_o); } @@ -1743,6 +1782,7 @@ dummy_func( } PyStackRef_CLOSE(values[i]); } + DEAD(values); if (err != 0) { Py_DECREF(set_o); ERROR_IF(true, error); @@ -1767,7 +1807,6 @@ dummy_func( } inst(SETUP_ANNOTATIONS, (--)) { - int err; PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, @@ -1775,7 +1814,8 @@ dummy_func( ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ - ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + ERROR_IF(err < 0, error); if (ann_dict == NULL) { ann_dict = PyDict_New(); ERROR_IF(ann_dict == NULL, error); @@ -1869,7 +1909,10 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - ERROR_IF(err, error); + if (err) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about @@ -1894,9 +1937,10 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(super == NULL, error); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); + PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); - ERROR_IF(PyStackRef_IsNull(attr), error); + ERROR_IF(attr_o == NULL, error); + attr = PyStackRef_FromPyObjectSteal(attr_o); null = PyStackRef_NULL; } @@ -1940,6 +1984,7 @@ dummy_func( } if (method_found) { self_or_null = self_st; // transfer ownership + DEAD(self_st); } else { PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; @@ -1991,6 +2036,7 @@ dummy_func( */ assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership + DEAD(owner); } else { /* meth is not an unbound method (but a regular attr, or @@ -2009,6 +2055,8 @@ dummy_func( attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); ERROR_IF(attr_o == NULL, error); + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -2170,9 +2218,9 @@ dummy_func( DEOPT_IF(code->co_argcount != 1); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; + DEAD(owner); } macro(LOAD_ATTR_PROPERTY) = @@ -2202,11 +2250,12 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; + DEAD(owner); new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -2255,7 +2304,6 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *old_value; - uint64_t new_version; DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); @@ -2265,9 +2313,8 @@ dummy_func( } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; - new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); ep->me_value = PyStackRef_AsPyObjectSteal(value); - dict->ma_version_tag = new_version; // PEP 509 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -2353,8 +2400,10 @@ dummy_func( double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + DEAD(left); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } @@ -2373,8 +2422,10 @@ dummy_func( Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + DEAD(left); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } @@ -2387,8 +2438,10 @@ dummy_func( STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + DEAD(left); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + DEAD(right); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); @@ -2571,6 +2624,14 @@ dummy_func( JUMP_BACKWARD_NO_INTERRUPT, }; + pseudo(JUMP_IF_FALSE, (cond -- cond)) = [ + COPY, TO_BOOL, POP_JUMP_IF_FALSE, + ]; + + pseudo(JUMP_IF_TRUE, (cond -- cond)) = [ + COPY, TO_BOOL, POP_JUMP_IF_TRUE, + ]; + tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -2602,6 +2663,7 @@ dummy_func( replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); + DEAD(cond); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -2611,6 +2673,7 @@ dummy_func( replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); + DEAD(cond); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -2620,6 +2683,7 @@ dummy_func( op(_IS_NONE, (value -- b)) { if (PyStackRef_Is(value, PyStackRef_None)) { b = PyStackRef_True; + DEAD(value); } else { b = PyStackRef_False; @@ -2692,9 +2756,10 @@ dummy_func( inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); DECREF_INPUTS(); - ERROR_IF(PyStackRef_IsNull(iter), error); + ERROR_IF(iter_o == NULL, error); + iter = PyStackRef_FromPyObjectSteal(iter_o); } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { @@ -2711,13 +2776,16 @@ dummy_func( ERROR_NO_POP(); } iter = iterable; + DEAD(iterable); } else if (PyGen_CheckExact(iterable_o)) { iter = iterable; + DEAD(iterable); } else { /* `iterable` is not a generator. */ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + DEAD(iterable); if (PyStackRef_IsNull(iter)) { ERROR_NO_POP(); } @@ -2755,7 +2823,6 @@ dummy_func( PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { - next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); if (!matches) { @@ -3002,16 +3069,18 @@ dummy_func( PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; - attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); - if (PyStackRef_IsNull(attr)) { + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + if (attr_o == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); } + ERROR_IF(true, error); } - ERROR_IF(PyStackRef_IsNull(attr), error); - self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); } inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { @@ -3042,9 +3111,10 @@ dummy_func( (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); - res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); - ERROR_IF(PyStackRef_IsNull(res), error); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { @@ -3071,7 +3141,7 @@ dummy_func( NOP, }; - inst(PUSH_EXC_INFO, (new_exc -- prev_exc, new_exc)) { + inst(PUSH_EXC_INFO, (exc -- prev_exc, new_exc)) { _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { @@ -3080,8 +3150,10 @@ dummy_func( else { prev_exc = PyStackRef_None; } - assert(PyStackRef_ExceptionInstanceCheck(new_exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + DEAD(exc); } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { @@ -3104,6 +3176,7 @@ dummy_func( assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; + DEAD(owner); } macro(LOAD_ATTR_METHOD_WITH_VALUES) = @@ -3121,6 +3194,7 @@ dummy_func( assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; + DEAD(owner); } macro(LOAD_ATTR_METHOD_NO_DICT) = @@ -3173,6 +3247,7 @@ dummy_func( assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; + DEAD(owner); } macro(LOAD_ATTR_METHOD_LAZY_DICT) = @@ -3207,11 +3282,11 @@ dummy_func( CALL_NON_PY_GENERAL, }; - specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null[1], args[oparg] -- callable, self_or_null[1], args[oparg])) { + specializing op(_SPECIALIZE_CALL, (counter/1, callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -3219,23 +3294,21 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null[1], args[oparg] -- func, maybe_self[1], args[oparg])) { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; - func = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - } - else { - func = callable; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } } // When calling Python, inline the call using DISPATCH_INLINED(). - op(_DO_CALL, (callable, self_or_null[1], args[oparg] -- res)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_DO_CALL, (callable[1], self_or_null[1], args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -3251,11 +3324,11 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + SYNC_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -3267,10 +3340,11 @@ dummy_func( /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + DEAD(self_or_null); ERROR_IF(true, error); } PyObject *res_o = PyObject_Vectorcall( @@ -3296,17 +3370,18 @@ dummy_func( } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + DEAD(self_or_null); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } - op(_MONITOR_CALL, (func, maybe_self[1], args[oparg] -- func, maybe_self[1], args[oparg])) { + op(_MONITOR_CALL, (func[1], maybe_self[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); @@ -3317,6 +3392,7 @@ dummy_func( else { arg0 = &_PyInstrumentation_MISSING; } + SYNC_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 @@ -3327,8 +3403,8 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; - op(_PY_FRAME_GENERAL, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_PY_FRAME_GENERAL, (callable[1], self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -3339,20 +3415,21 @@ dummy_func( assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, args, total_args, NULL, frame ); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + // The frame has stolen all the arguments from the stack. + INPUTS_DEAD(); SYNC_SP(); - if (new_frame == NULL) { + if (temp == NULL) { ERROR_NO_POP(); } + new_frame = temp; } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null[1], unused[oparg] -- callable, self_or_null[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; EXIT_IF(func->func_version != func_version); @@ -3366,8 +3443,8 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null[1], unused[oparg] -- callable, null[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_METHOD_VERSION, (func_version/2, callable[1], null[1], unused[oparg] -- callable[1], null[1], unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); PyObject *func = ((PyMethodObject *)callable_o)->im_func; @@ -3376,15 +3453,16 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null[0])); } - op(_EXPAND_METHOD, (callable, null[1], unused[oparg] -- method, self[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - + op(_EXPAND_METHOD, (callable[1], null[1], unused[oparg] -- method[1], self[1], unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyStackRef_IsNull(null[0])); + DEAD(null); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); } macro(CALL_BOUND_METHOD_GENERAL) = @@ -3397,17 +3475,17 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused[1], unused[oparg] -- callable, unused[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_IS_NOT_PY_CALLABLE, (callable[1], unused[1], unused[oparg] -- callable[1], unused[1], unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_NON_PY_GENERAL, (callable, self_or_null[1], args[oparg] -- res)) { + op(_CALL_NON_PY_GENERAL, (callable[1], self_or_null[1], args[oparg] -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3426,10 +3504,11 @@ dummy_func( NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + DEAD(self_or_null); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3441,61 +3520,64 @@ dummy_func( _CALL_NON_PY_GENERAL + _CHECK_PERIODIC; - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null[1], unused[oparg] -- callable, null[1], unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable[1], null[1], unused[oparg] -- callable[1], null[1], unused[oparg])) { EXIT_IF(!PyStackRef_IsNull(null[0])); - EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null[1], unused[oparg] -- func, self[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable[1], null[1], unused[oparg] -- func[1], self[1], unused[oparg])) { + DEAD(null); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); } op(_CHECK_PEP_523, (--)) { DEOPT_IF(tstate->interp->eval_frame); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null[1], unused[oparg] -- callable, self_or_null[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_FUNCTION_EXACT_ARGS, (callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))); } - op(_CHECK_STACK_SPACE, (callable, self_or_null[1], unused[oparg] -- callable, self_or_null[1], unused[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_STACK_SPACE, (callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable[1], self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } + INPUTS_DEAD(); } op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- )) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + DEAD(new_frame); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -3531,7 +3613,9 @@ dummy_func( assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null)); + DEAD(null); DEOPT_IF(callable_o != (PyObject *)&PyType_Type); + DEAD(callable); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3546,6 +3630,8 @@ dummy_func( DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + DEAD(null); + DEAD(callable); PyStackRef_CLOSE(arg); ERROR_IF(PyStackRef_IsNull(res), error); } @@ -3565,6 +3651,8 @@ dummy_func( DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + DEAD(null); + DEAD(callable); PyStackRef_CLOSE(arg); ERROR_IF(PyStackRef_IsNull(res), error); } @@ -3575,9 +3663,9 @@ dummy_func( _CALL_TUPLE_1 + _CHECK_PERIODIC; - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - DEOPT_IF(!PyStackRef_IsNull(null)); + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], null[1], args[oparg] -- init[1], self[1], args[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0])); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != type_version); @@ -3587,24 +3675,26 @@ dummy_func( PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); - if (PyStackRef_IsNull(self)) { + PyObject *self_o = _PyType_NewManagedObject(tp); + if (self_o == NULL) { ERROR_NO_POP(); } - PyStackRef_CLOSE(callable); - init = PyStackRef_FromPyObjectNew(init_func); + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) { + op(_CREATE_INIT_FRAME, (init[1], self[1], args[oparg] -- init_frame: _PyInterpreterFrame *)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self); - PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); - args[-1] = self; + shim->localsplus[0] = PyStackRef_DUP(self[0]); + DEAD(init); + DEAD(self); init_frame = _PyEvalFramePushAndInit( - tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); SYNC_SP(); if (init_frame == NULL) { _PyEval_FrameClearAndPop(tstate, shim); @@ -3632,16 +3722,18 @@ dummy_func( Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); ERROR_NO_POP(); } + DEAD(should_be_none); } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null[1], args[oparg] -- res)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CALL_BUILTIN_CLASS, (callable[1], self_or_null[1], args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } + DEAD(self_or_null); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL); @@ -3657,7 +3749,7 @@ dummy_func( for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3668,9 +3760,9 @@ dummy_func( _CALL_BUILTIN_CLASS + _CHECK_PERIODIC; - op(_CALL_BUILTIN_O, (callable, self_or_null[1], args[oparg] -- res)) { + op(_CALL_BUILTIN_O, (callable[1], self_or_null[1], args[oparg] -- res)) { /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3691,7 +3783,9 @@ dummy_func( assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable); + DEAD(args); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3702,15 +3796,16 @@ dummy_func( _CALL_BUILTIN_O + _CHECK_PERIODIC; - op(_CALL_BUILTIN_FAST, (callable, self_or_null[1], args[oparg] -- res)) { + op(_CALL_BUILTIN_FAST, (callable[1], self_or_null[1], args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } + DEAD(self_or_null); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); @@ -3732,7 +3827,7 @@ dummy_func( for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3743,9 +3838,9 @@ dummy_func( _CALL_BUILTIN_FAST + _CHECK_PERIODIC; - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null[1], args[oparg] -- res)) { + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable[1], self_or_null[1], args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3774,7 +3869,8 @@ dummy_func( for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3785,9 +3881,9 @@ dummy_func( _CALL_BUILTIN_FAST_WITH_KEYWORDS + _CHECK_PERIODIC; - inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null[1], args[oparg] -- res)) { + inst(CALL_LEN, (unused/1, unused/2, callable[1], self_or_null[1], args[oparg] -- res)) { /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3809,14 +3905,14 @@ dummy_func( if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null[1], args[oparg] -- res)) { + inst(CALL_ISINSTANCE, (unused/1, unused/2, callable[1], self_or_null[1], args[oparg] -- res)) { /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3837,7 +3933,7 @@ dummy_func( assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(inst_stackref); PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); } // This is secretly a super-instruction @@ -3863,8 +3959,8 @@ dummy_func( #endif } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null[1], args[oparg] -- res)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CALL_METHOD_DESCRIPTOR_O, (callable[1], self_or_null[1], args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3893,7 +3989,9 @@ dummy_func( assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable); + DEAD(args); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3904,8 +4002,8 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_O + _CHECK_PERIODIC; - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null[1], args[oparg] -- res)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable[1], self_or_null[1], args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3921,14 +4019,14 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); } + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3937,7 +4035,8 @@ dummy_func( for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3948,9 +4047,9 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + _CHECK_PERIODIC; - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null[1], args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable[1], self_or_null[1], args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3974,7 +4073,9 @@ dummy_func( _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable); + DEAD(args); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3985,8 +4086,8 @@ dummy_func( _CALL_METHOD_DESCRIPTOR_NOARGS + _CHECK_PERIODIC; - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null[1], args[oparg] -- res)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable[1], self_or_null[1], args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -4001,15 +4102,15 @@ dummy_func( PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); } + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4018,7 +4119,8 @@ dummy_func( for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4050,8 +4152,22 @@ dummy_func( GO_TO_INSTRUCTION(CALL_KW); } - op(_DO_CALL_KW, (callable, self_or_null[1], args[oparg], kwnames -- res)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + DEAD(kwnames_in); + } + + op(_DO_CALL_KW, (callable[1], self_or_null[1], args[oparg], kwnames -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: @@ -4060,17 +4176,6 @@ dummy_func( args--; total_args++; } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && @@ -4080,12 +4185,12 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + SYNC_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -4125,16 +4230,17 @@ dummy_func( } PyStackRef_CLOSE(kwnames); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + DEAD(self_or_null); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } - op(_PY_FRAME_KW, (callable, self_or_null[1], args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_PY_FRAME_KW, (callable[1], self_or_null[1], args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -4148,7 +4254,7 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -4160,8 +4266,8 @@ dummy_func( } } - op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, self_or_null[1], unused[oparg], kwnames -- callable, self_or_null[1], unused[oparg], kwnames)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable[1], self_or_null[1], unused[oparg], kwnames -- callable[1], self_or_null[1], unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; EXIT_IF(func->func_version != func_version); @@ -4175,8 +4281,8 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null[1], unused[oparg], kwnames -- callable, null[1], unused[oparg], kwnames)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable[1], null[1], unused[oparg], kwnames -- callable[1], null[1], unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); PyObject *func = ((PyMethodObject *)callable_o)->im_func; @@ -4185,15 +4291,16 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null[0])); } - op(_EXPAND_METHOD_KW, (callable, null[1], unused[oparg], kwnames -- method, self[1], unused[oparg], kwnames)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_EXPAND_METHOD_KW, (callable[1], null[1], unused[oparg], unused -- method[1], self[1], unused[oparg], unused)) { + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); } macro(CALL_KW_BOUND_METHOD) = @@ -4206,11 +4313,11 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null[1], args[oparg], kwnames -- callable, self_or_null[1], args[oparg], kwnames)) { + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable[1], self_or_null[1], args[oparg], kwnames -- callable[1], self_or_null[1], args[oparg], kwnames)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -4221,20 +4328,21 @@ dummy_func( macro(CALL_KW) = _SPECIALIZE_CALL_KW + unused/2 + + _MAYBE_EXPAND_METHOD_KW + _DO_CALL_KW; - op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused[1], unused[oparg], kwnames -- callable, unused[1], unused[oparg], kwnames)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable[1], unused[1], unused[oparg], kwnames -- callable[1], unused[1], unused[oparg], kwnames)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_KW_NON_PY, (callable, self_or_null[1], args[oparg], kwnames -- res)) { + op(_CALL_KW_NON_PY, (callable[1], self_or_null[1], args[oparg], kwnames -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -4256,10 +4364,11 @@ dummy_func( PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + DEAD(self_or_null); + PyStackRef_CLOSE(callable[0]); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4275,7 +4384,29 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { + op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + DEAD(callargs); + } + else { + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + if (err < 0) { + ERROR_NO_POP(); + } + PyObject *tuple_o = PySequence_Tuple(callargs_o); + if (tuple_o == NULL) { + ERROR_NO_POP(); + } + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } + kwargs_out = kwargs_in; + DEAD(kwargs_in); + } + + op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); @@ -4283,32 +4414,23 @@ dummy_func( // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - int err = check_args_iterable(tstate, func, callargs); - if (err < 0) { - ERROR_NO_POP(); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - ERROR_NO_POP(); - } - PyStackRef_CLOSE(callargs_st); - callargs_st = PyStackRef_FromPyObjectSteal(tuple); - callargs = tuple; - } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) ERROR_NO_POP(); - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + if (err) { + ERROR_NO_POP(); + } + result_o = PyObject_Call(func, callargs, kwargs); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (PyStackRef_IsNull(result)) { + if (result_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); @@ -4318,7 +4440,7 @@ dummy_func( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); if (err < 0) { - PyStackRef_CLEAR(result); + Py_CLEAR(result_o); } } } @@ -4332,11 +4454,12 @@ dummy_func( int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs, frame); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + INPUTS_DEAD(); + SYNC_SP(); if (new_frame == NULL) { ERROR_NO_POP(); } @@ -4344,14 +4467,18 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + result_o = PyObject_Call(func, callargs, kwargs); } - DECREF_INPUTS(); - assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); - ERROR_IF(PyStackRef_IsNull(result), error); + PyStackRef_XCLOSE(kwargs_st); + DEAD(kwargs_st); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + ERROR_IF(result_o == NULL, error); + result = PyStackRef_FromPyObjectSteal(result_o); } macro(CALL_FUNCTION_EX) = + _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + _CHECK_PERIODIC; @@ -4363,59 +4490,33 @@ dummy_func( PyFunction_New(codeobj, GLOBALS()); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) { - ERROR_NO_POP(); - } + ERROR_IF(func_obj == NULL, error); _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); } - inst(SET_FUNCTION_ATTRIBUTE, (attr_st, func_st -- func_st)) { - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); - + inst(SET_FUNCTION_ATTRIBUTE, (attr_st, func_in -- func_out)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + DEAD(func_in); assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - case MAKE_FUNCTION_ANNOTATE: - assert(PyCallable_Check(attr)); - assert(func_obj->func_annotate == NULL); - func_obj->func_annotate = attr; - break; - default: - Py_UNREACHABLE(); - } + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; } inst(RETURN_GENERATOR, (-- res)) { - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - ERROR_NO_POP(); - } + ERROR_IF(gen == NULL, error); assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_STACK(); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -4423,12 +4524,12 @@ dummy_func( gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); - LOAD_SP(); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); LLTRACE_RESUME_FRAME(); } @@ -4458,12 +4559,14 @@ dummy_func( /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value_o)) { - res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyObject *res_o = PyObject_Format(value_o, NULL); PyStackRef_CLOSE(value); - ERROR_IF(PyStackRef_IsNull(res), error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; + DEAD(value); } } @@ -4507,8 +4610,12 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; - pure inst(SWAP, (bottom, unused[oparg-2], top -- - top, unused[oparg-2], bottom)) { + pure inst(SWAP, (bottom_in, unused[oparg-2], top_in -- + top_out, unused[oparg-2], bottom_out)) { + bottom_out = bottom_in; + DEAD(bottom_in); + top_out = top_in; + DEAD(top_in); assert(oparg >= 2); } @@ -4519,10 +4626,8 @@ dummy_func( original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; next_instr = this_instr; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; goto error; @@ -4645,29 +4750,34 @@ dummy_func( ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { + int is_true = PyStackRef_Is(flag, PyStackRef_True); + DEAD(flag); SYNC_SP(); - EXIT_IF(!PyStackRef_Is(flag, PyStackRef_True)); - assert(PyStackRef_Is(flag, PyStackRef_True)); + EXIT_IF(!is_true); } op (_GUARD_IS_FALSE_POP, (flag -- )) { + int is_false = PyStackRef_Is(flag, PyStackRef_False); + DEAD(flag); SYNC_SP(); - EXIT_IF(!PyStackRef_Is(flag, PyStackRef_False)); - assert(PyStackRef_Is(flag, PyStackRef_False)); + EXIT_IF(!is_false); } op (_GUARD_IS_NONE_POP, (val -- )) { - SYNC_SP(); - if (!PyStackRef_Is(val, PyStackRef_None)) { + int is_none = PyStackRef_Is(val, PyStackRef_None); + if (!is_none) { PyStackRef_CLOSE(val); + SYNC_SP(); EXIT_IF(1); } + DEAD(val); } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { - SYNC_SP(); - EXIT_IF(PyStackRef_Is(val, PyStackRef_None)); + int is_none = PyStackRef_Is(val, PyStackRef_None); PyStackRef_CLOSE(val); + SYNC_SP(); + EXIT_IF(is_none); } op(_JUMP_TO_TOP, (--)) { @@ -4703,7 +4813,7 @@ dummy_func( printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } @@ -4771,14 +4881,36 @@ dummy_func( } tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { - assert(PyFunction_Check(frame->f_funcobj)); - DEOPT_IF(((PyFunctionObject *)frame->f_funcobj)->func_version != func_version); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + DEOPT_IF(func->func_version != func_version); } + tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; + } + + tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { + PyDictObject *dict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; + } + /* Internal -- for testing executors */ op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); exe->count++; + DEAD(opt); } tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { @@ -4791,7 +4923,7 @@ dummy_func( printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } @@ -4831,6 +4963,14 @@ dummy_func( assert(((_PyExecutorObject *)executor)->vm_data.valid); } + tier2 op(_MAKE_WARM, (--)) { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + } + tier2 op(_FATAL_ERROR, (--)) { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/ceval.c b/Python/ceval.c index 44b39f5d36c93c..f4e0add3034707 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -196,7 +196,7 @@ lltrace_instruction(_PyInterpreterFrame *frame, static void lltrace_resume_frame(_PyInterpreterFrame *frame) { - PyObject *fobj = frame->f_funcobj; + PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (!PyStackRef_CodeCheck(frame->f_executable) || fobj == NULL || !PyFunction_Check(fobj) @@ -274,10 +274,9 @@ static void monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); #ifdef HAVE_ERRNO_H @@ -394,6 +393,13 @@ const _Py_SpecialMethod _Py_SpecialMethods[] = { } }; +const size_t _Py_FunctionAttributeOffsets[] = { + [MAKE_FUNCTION_CLOSURE] = offsetof(PyFunctionObject, func_closure), + [MAKE_FUNCTION_ANNOTATIONS] = offsetof(PyFunctionObject, func_annotations), + [MAKE_FUNCTION_KWDEFAULTS] = offsetof(PyFunctionObject, func_kwdefaults), + [MAKE_FUNCTION_DEFAULTS] = offsetof(PyFunctionObject, func_defaults), + [MAKE_FUNCTION_ANNOTATE] = offsetof(PyFunctionObject, func_annotate), +}; // PEP 634: Structural Pattern Matching @@ -781,7 +787,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_DEBUG /* Set these to invalid but identifiable values for debugging. */ - entry_frame.f_funcobj = (PyObject*)0xaaa0; + entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry_frame.f_locals = (PyObject*)0xaaa1; entry_frame.frame_obj = (PyFrameObject*)0xaaa2; entry_frame.f_globals = (PyObject*)0xaaa3; @@ -1036,6 +1042,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uopcode = next_uop->opcode; #ifdef Py_DEBUG if (lltrace >= 3) { + dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } @@ -1043,8 +1050,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int printf("%4d uop: ", (int)(next_uop - current_executor->trace)); } _PyUOpPrint(next_uop); - printf(" stack_level=%d\n", - (int)(stack_pointer - _PyFrame_Stackbase(frame))); + printf("\n"); } #endif next_uop++; @@ -1281,7 +1287,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, } else { /* This will not fail. */ - kwonly_sig = PyUnicode_FromString(""); + kwonly_sig = Py_GetConstant(Py_CONSTANT_EMPTY_STR); assert(kwonly_sig != NULL); } _PyErr_Format(tstate, PyExc_TypeError, @@ -1719,18 +1725,19 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) /* Consumes references to func, locals and all the args */ _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { - PyCodeObject * code = (PyCodeObject *)func->func_code; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + PyCodeObject * code = (PyCodeObject *)func_obj->func_code; CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); if (frame == NULL) { goto fail; } _PyFrame_Initialize(frame, func, locals, code, 0, previous); - if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { + if (initialize_locals(tstate, func_obj, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); return NULL; @@ -1738,7 +1745,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return frame; fail: /* Consume the references */ - Py_DECREF(func); + PyStackRef_CLOSE(func); Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { PyStackRef_CLOSE(args[i]); @@ -1754,7 +1761,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, } static _PyInterpreterFrame * -_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, _PyStackRef func, PyObject *locals, PyObject *const* args, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { @@ -1784,7 +1791,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, Steals references to func, callargs and kwargs. */ static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); @@ -1793,7 +1800,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, if (has_dict) { newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); if (newargs == NULL) { - Py_DECREF(func); + PyStackRef_CLOSE(func); goto error; } } @@ -1805,7 +1812,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( - tstate, (PyFunctionObject *)func, locals, + tstate, func, locals, newargs, nargs, kwnames, previous ); if (has_dict) { @@ -1831,7 +1838,6 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, { /* _PyEvalFramePushAndInit consumes the references * to func, locals and all its arguments */ - Py_INCREF(func); Py_XINCREF(locals); for (size_t i = 0; i < argcount; i++) { Py_INCREF(args[i]); @@ -1843,7 +1849,8 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( - tstate, func, locals, args, argcount, kwnames, NULL); + tstate, PyStackRef_FromPyObjectNew(func), locals, + args, argcount, kwnames, NULL); if (frame == NULL) { return NULL; } @@ -2919,11 +2926,11 @@ _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right) return 0; } -static int -check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args) +int +_Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args) { if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) { - /* check_args_iterable() may be called with a live exception: + /* _Py_Check_ArgsIterable() may be called with a live exception: * clear it to prevent calling _PyObject_FunctionStr() with an * exception set. */ _PyErr_Clear(tstate); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6f4476d055b5ec..4c9f59f837e11b 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -7,6 +7,7 @@ #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystats.h" // _Py_PrintSpecializationStats() +#include "pycore_pythread.h" // PyThread_hang_thread() /* Notes about the implementation: @@ -277,10 +278,9 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) /* Take the GIL. The function saves errno at entry and restores its value at exit. + It may hang rather than return if the interpreter has been finalized. - tstate must be non-NULL. - - Returns 1 if the GIL was acquired, or 0 if not. */ + tstate must be non-NULL. */ static void take_gil(PyThreadState *tstate) { @@ -293,12 +293,18 @@ take_gil(PyThreadState *tstate) if (_PyThreadState_MustExit(tstate)) { /* bpo-39877: If Py_Finalize() has been called and tstate is not the - thread which called Py_Finalize(), exit immediately the thread. + thread which called Py_Finalize(), this thread cannot continue. This code path can be reached by a daemon thread after Py_Finalize() completes. In this case, tstate is a dangling pointer: points to - PyThreadState freed memory. */ - PyThread_exit_thread(); + PyThreadState freed memory. + + This used to call a *thread_exit API, but that was not safe as it + lacks stack unwinding and local variable destruction important to + C++. gh-87135: The best that can be done is to hang the thread as + the public APIs calling this have no error reporting mechanism (!). + */ + PyThread_hang_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); @@ -342,7 +348,9 @@ take_gil(PyThreadState *tstate) if (drop_requested) { _Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); } - PyThread_exit_thread(); + // gh-87135: hang the thread as *thread_exit() is not a safe + // API. It lacks stack unwind and local variable destruction. + PyThread_hang_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); @@ -383,7 +391,7 @@ take_gil(PyThreadState *tstate) if (_PyThreadState_MustExit(tstate)) { /* bpo-36475: If Py_Finalize() has been called and tstate is not - the thread which called Py_Finalize(), exit immediately the + the thread which called Py_Finalize(), gh-87135: hang the thread. This code path can be reached by a daemon thread which was waiting @@ -393,7 +401,7 @@ take_gil(PyThreadState *tstate) /* tstate could be a dangling pointer, so don't pass it to drop_gil(). */ drop_gil(interp, NULL, 1); - PyThread_exit_thread(); + PyThread_hang_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); @@ -1289,6 +1297,12 @@ _Py_HandlePending(PyThreadState *tstate) _Py_RunGC(tstate); } + if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + _Py_Executors_InvalidateCold(tstate->interp); + tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + } + /* GIL drop request */ if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9e1540674d4219..6674c4ccf9f693 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -108,6 +108,7 @@ do { \ /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ + assert(frame->stackpointer == NULL); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ @@ -326,26 +327,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { " in enclosing scope" #define NAME_ERROR_MSG "name '%.200s' is not defined" -#define DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dval, result) \ -do { \ - if (Py_REFCNT(left) == 1) { \ - ((PyFloatObject *)left)->ob_fval = (dval); \ - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);\ - result = (left); \ - } \ - else if (Py_REFCNT(right) == 1) {\ - ((PyFloatObject *)right)->ob_fval = (dval); \ - _Py_DECREF_NO_DEALLOC(left); \ - result = (right); \ - }\ - else { \ - result = PyFloat_FromDouble(dval); \ - if ((result) == NULL) GOTO_ERROR(error); \ - _Py_DECREF_NO_DEALLOC(left); \ - _Py_DECREF_NO_DEALLOC(right); \ - } \ -} while (0) - // If a trace function sets a new f_lineno and // *then* raises, we use the destination when searching // for an exception handler, displaying the traceback, and so on diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 8dae747c44a543..9b3373bc1a67a5 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -36,6 +36,33 @@ monitoring_use_tool_id(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(monitoring_clear_tool_id__doc__, +"clear_tool_id($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_CLEAR_TOOL_ID_METHODDEF \ + {"clear_tool_id", (PyCFunction)monitoring_clear_tool_id, METH_O, monitoring_clear_tool_id__doc__}, + +static PyObject * +monitoring_clear_tool_id_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_clear_tool_id(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_clear_tool_id_impl(module, tool_id); + +exit: + return return_value; +} + PyDoc_STRVAR(monitoring_free_tool_id__doc__, "free_tool_id($module, tool_id, /)\n" "--\n" @@ -304,4 +331,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=14ffc0884a6de50a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f81876c6aba9be8 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index 9c0a3fad314cb5..2cb3875db35058 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,6 +16,12 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +static const char *codecs_builtin_error_handlers[] = { + "strict", "ignore", "replace", + "xmlcharrefreplace", "backslashreplace", "namereplace", + "surrogatepass", "surrogateescape", +}; + const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ @@ -618,6 +624,20 @@ int PyCodec_RegisterError(const char *name, PyObject *error) name, error); } +int _PyCodec_UnregisterError(const char *name) +{ + for (size_t i = 0; i < Py_ARRAY_LENGTH(codecs_builtin_error_handlers); ++i) { + if (strcmp(name, codecs_builtin_error_handlers[i]) == 0) { + PyErr_Format(PyExc_ValueError, + "cannot un-register built-in error handler '%s'", name); + return -1; + } + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->codecs.initialized); + return PyDict_PopString(interp->codecs.error_registry, name, NULL); +} + /* Lookup the error handling callback function registered under the name error. As a special case NULL can be passed, in which case the error handling callback for strict encoding will be returned. */ @@ -676,7 +696,7 @@ PyObject *PyCodec_IgnoreErrors(PyObject *exc) wrong_exception_type(exc); return NULL; } - return Py_BuildValue("(Nn)", PyUnicode_New(0, 0), end); + return Py_BuildValue("(Nn)", Py_GetConstant(Py_CONSTANT_EMPTY_STR), end); } @@ -1470,6 +1490,8 @@ _PyCodec_InitRegistry(PyInterpreterState *interp) } } }; + // ensure that the built-in error handlers' names are kept in sync + assert(Py_ARRAY_LENGTH(methods) == Py_ARRAY_LENGTH(codecs_builtin_error_handlers)); assert(interp->codecs.initialized == 0); interp->codecs.search_path = PyList_New(0); diff --git a/Python/codegen.c b/Python/codegen.c index 0305f4299aec56..689d2b5124e9d3 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -194,6 +194,7 @@ static int codegen_visit_expr(compiler *, expr_ty); static int codegen_augassign(compiler *, stmt_ty); static int codegen_annassign(compiler *, stmt_ty); static int codegen_subscript(compiler *, expr_ty); +static int codegen_slice_two_parts(compiler *, expr_ty); static int codegen_slice(compiler *, expr_ty); static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); @@ -3140,17 +3141,15 @@ codegen_boolop(compiler *c, expr_ty e) location loc = LOC(e); assert(e->kind == BoolOp_kind); if (e->v.BoolOp.op == And) - jumpi = POP_JUMP_IF_FALSE; + jumpi = JUMP_IF_FALSE; else - jumpi = POP_JUMP_IF_TRUE; + jumpi = JUMP_IF_TRUE; NEW_JUMP_TARGET_LABEL(c, end); s = e->v.BoolOp.values; n = asdl_seq_LEN(s) - 1; assert(n >= 0); for (i = 0; i < n; ++i) { VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i)); - ADDOP_I(c, loc, COPY, 1); - ADDOP(c, loc, TO_BOOL); ADDOP_JUMP(c, loc, jumpi, end); ADDOP(c, loc, POP_TOP); } @@ -5007,12 +5006,8 @@ codegen_visit_expr(compiler *c, expr_ty e) } break; case Slice_kind: - { - int n = codegen_slice(c, e); - RETURN_IF_ERROR(n); - ADDOP_I(c, loc, BUILD_SLICE, n); + RETURN_IF_ERROR(codegen_slice(c, e)); break; - } case Name_kind: return codegen_nameop(c, loc, e->v.Name.id, e->v.Name.ctx); /* child nodes of List and Tuple will have expr_context set */ @@ -5025,9 +5020,22 @@ codegen_visit_expr(compiler *c, expr_ty e) } static bool -is_two_element_slice(expr_ty s) +is_constant_slice(expr_ty s) { return s->kind == Slice_kind && + (s->v.Slice.lower == NULL || + s->v.Slice.lower->kind == Constant_kind) && + (s->v.Slice.upper == NULL || + s->v.Slice.upper->kind == Constant_kind) && + (s->v.Slice.step == NULL || + s->v.Slice.step->kind == Constant_kind); +} + +static bool +should_apply_two_element_slice_optimization(expr_ty s) +{ + return !is_constant_slice(s) && + s->kind == Slice_kind && s->v.Slice.step == NULL; } @@ -5048,8 +5056,8 @@ codegen_augassign(compiler *c, stmt_ty s) break; case Subscript_kind: VISIT(c, expr, e->v.Subscript.value); - if (is_two_element_slice(e->v.Subscript.slice)) { - RETURN_IF_ERROR(codegen_slice(c, e->v.Subscript.slice)); + if (should_apply_two_element_slice_optimization(e->v.Subscript.slice)) { + RETURN_IF_ERROR(codegen_slice_two_parts(c, e->v.Subscript.slice)); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); ADDOP_I(c, loc, COPY, 3); @@ -5086,7 +5094,7 @@ codegen_augassign(compiler *c, stmt_ty s) ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); break; case Subscript_kind: - if (is_two_element_slice(e->v.Subscript.slice)) { + if (should_apply_two_element_slice_optimization(e->v.Subscript.slice)) { ADDOP_I(c, loc, SWAP, 4); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, SWAP, 2); @@ -5233,8 +5241,10 @@ codegen_subscript(compiler *c, expr_ty e) } VISIT(c, expr, e->v.Subscript.value); - if (is_two_element_slice(e->v.Subscript.slice) && ctx != Del) { - RETURN_IF_ERROR(codegen_slice(c, e->v.Subscript.slice)); + if (should_apply_two_element_slice_optimization(e->v.Subscript.slice) && + ctx != Del + ) { + RETURN_IF_ERROR(codegen_slice_two_parts(c, e->v.Subscript.slice)); if (ctx == Load) { ADDOP(c, loc, BINARY_SLICE); } @@ -5256,15 +5266,9 @@ codegen_subscript(compiler *c, expr_ty e) return SUCCESS; } -/* Returns the number of the values emitted, - * thus are needed to build the slice, or -1 if there is an error. */ static int -codegen_slice(compiler *c, expr_ty s) +codegen_slice_two_parts(compiler *c, expr_ty s) { - int n = 2; - assert(s->kind == Slice_kind); - - /* only handles the cases where BUILD_SLICE is emitted */ if (s->v.Slice.lower) { VISIT(c, expr, s->v.Slice.lower); } @@ -5279,11 +5283,45 @@ codegen_slice(compiler *c, expr_ty s) ADDOP_LOAD_CONST(c, LOC(s), Py_None); } + return 0; +} + +static int +codegen_slice(compiler *c, expr_ty s) +{ + int n = 2; + assert(s->kind == Slice_kind); + + if (is_constant_slice(s)) { + PyObject *start = NULL; + if (s->v.Slice.lower) { + start = s->v.Slice.lower->v.Constant.value; + } + PyObject *stop = NULL; + if (s->v.Slice.upper) { + stop = s->v.Slice.upper->v.Constant.value; + } + PyObject *step = NULL; + if (s->v.Slice.step) { + step = s->v.Slice.step->v.Constant.value; + } + PyObject *slice = PySlice_New(start, stop, step); + if (slice == NULL) { + return ERROR; + } + ADDOP_LOAD_CONST_NEW(c, LOC(s), slice); + return SUCCESS; + } + + RETURN_IF_ERROR(codegen_slice_two_parts(c, s)); + if (s->v.Slice.step) { n++; VISIT(c, expr, s->v.Slice.step); } - return n; + + ADDOP_I(c, LOC(s), BUILD_SLICE, n); + return SUCCESS; } diff --git a/Python/compile.c b/Python/compile.c index 7b3e6f336e44b1..d463fcde204a05 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -911,7 +911,17 @@ PyObject * _PyCompile_StaticAttributesAsTuple(compiler *c) { assert(c->u->u_static_attributes); - return PySequence_Tuple(c->u->u_static_attributes); + PyObject *static_attributes_unsorted = PySequence_List(c->u->u_static_attributes); + if (static_attributes_unsorted == NULL) { + return NULL; + } + if (PyList_Sort(static_attributes_unsorted) != 0) { + Py_DECREF(static_attributes_unsorted); + return NULL; + } + PyObject *static_attributes = PySequence_Tuple(static_attributes_unsorted); + Py_DECREF(static_attributes_unsorted); + return static_attributes; } int @@ -1524,7 +1534,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, _PyCompile_CodeUnitMetadata *umd = &c->u->u_metadata; -#define SET_MATADATA_INT(key, value) do { \ +#define SET_METADATA_INT(key, value) do { \ PyObject *v = PyLong_FromLong((long)value); \ if (v == NULL) goto finally; \ int res = PyDict_SetItemString(metadata, key, v); \ @@ -1532,10 +1542,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (res < 0) goto finally; \ } while (0); - SET_MATADATA_INT("argcount", umd->u_argcount); - SET_MATADATA_INT("posonlyargcount", umd->u_posonlyargcount); - SET_MATADATA_INT("kwonlyargcount", umd->u_kwonlyargcount); -#undef SET_MATADATA_INT + SET_METADATA_INT("argcount", umd->u_argcount); + SET_METADATA_INT("posonlyargcount", umd->u_posonlyargcount); + SET_METADATA_INT("kwonlyargcount", umd->u_kwonlyargcount); +#undef SET_METADATA_INT int addNone = mod->kind != Expression_kind; if (_PyCodegen_AddReturnAtEnd(c, addNone) < 0) { diff --git a/Python/context.c b/Python/context.c index e52efbb6516d5c..8bc487a33c890b 100644 --- a/Python/context.c +++ b/Python/context.c @@ -112,10 +112,11 @@ context_event_name(PyContextEvent event) { Py_UNREACHABLE(); } -static void notify_context_watchers(PyContextEvent event, PyContext *ctx) +static void +notify_context_watchers(PyThreadState *ts, PyContextEvent event, PyObject *ctx) { assert(Py_REFCNT(ctx) > 0); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = ts->interp; assert(interp->_initialized); uint8_t bits = interp->active_context_watchers; int i = 0; @@ -192,7 +193,7 @@ _PyContext_Enter(PyThreadState *ts, PyObject *octx) ts->context = Py_NewRef(ctx); ts->context_ver++; - notify_context_watchers(Py_CONTEXT_EVENT_ENTER, ctx); + notify_context_watchers(ts, Py_CONTEXT_EVENT_ENTER, octx); return 0; } @@ -226,7 +227,7 @@ _PyContext_Exit(PyThreadState *ts, PyObject *octx) return -1; } - notify_context_watchers(Py_CONTEXT_EVENT_EXIT, ctx); + notify_context_watchers(ts, Py_CONTEXT_EVENT_EXIT, octx); Py_SETREF(ts->context, (PyObject *)ctx->ctx_prev); ts->context_ver++; @@ -1154,48 +1155,31 @@ token_tp_dealloc(PyContextToken *self) static PyObject * token_tp_repr(PyContextToken *self) { - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - - if (_PyUnicodeWriter_WriteASCIIString(&writer, "tok_used) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, " used", 5) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, " used", 5) < 0) { goto error; } } - - if (_PyUnicodeWriter_WriteASCIIString(&writer, " var=", 5) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, " var=", 5) < 0) { goto error; } - - PyObject *var = PyObject_Repr((PyObject *)self->tok_var); - if (var == NULL) { + if (PyUnicodeWriter_WriteRepr(writer, (PyObject *)self->tok_var) < 0) { goto error; } - if (_PyUnicodeWriter_WriteStr(&writer, var) < 0) { - Py_DECREF(var); - goto error; - } - Py_DECREF(var); - - PyObject *addr = PyUnicode_FromFormat(" at %p>", self); - if (addr == NULL) { - goto error; - } - if (_PyUnicodeWriter_WriteStr(&writer, addr) < 0) { - Py_DECREF(addr); + if (PyUnicodeWriter_Format(writer, " at %p>", self) < 0) { goto error; } - Py_DECREF(addr); - - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 93ab068f9de949..15a6c7bc1a7966 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,9 +14,11 @@ case _CHECK_PERIODIC: { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) JUMP_TO_ERROR(); } break; @@ -28,7 +30,9 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) JUMP_TO_ERROR(); } } @@ -62,10 +66,12 @@ oparg = CURRENT_OPARG(); _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) JUMP_TO_ERROR(); } value = PyStackRef_DUP(value_s); @@ -323,11 +329,13 @@ case _END_SEND: { _PyStackRef value; _PyStackRef receiver; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; (void)receiver; + val = value; PyStackRef_CLOSE(receiver); - stack_pointer[-2] = value; + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -337,7 +345,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); @@ -360,7 +370,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) JUMP_TO_ERROR(); res = err ? PyStackRef_True : PyStackRef_False; @@ -390,7 +402,7 @@ } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortalLoose(value_o)); + assert(_Py_IsImmortal(value_o)); res = PyStackRef_False; } else { @@ -443,7 +455,7 @@ } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortalLoose(value_o)); + assert(_Py_IsImmortal(value_o)); res = PyStackRef_False; } else { @@ -469,7 +481,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); @@ -527,8 +541,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -547,8 +561,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -567,8 +581,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -629,8 +643,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -650,8 +664,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -671,8 +685,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -708,8 +722,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -723,7 +737,9 @@ _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; + #ifndef NDEBUG PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + #endif PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); int next_oparg; #if TIER_ONE @@ -750,11 +766,11 @@ * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - _Py_DECREF_NO_DEALLOC(left_o); + PyStackRef_CLOSE(left); PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) JUMP_TO_ERROR(); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -775,7 +791,9 @@ container = stack_pointer[-2]; PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) JUMP_TO_ERROR(); @@ -794,8 +812,10 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -803,8 +823,14 @@ res_o = NULL; } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(container); if (res_o == NULL) JUMP_TO_ERROR(); @@ -824,15 +850,23 @@ start = stack_pointer[-2]; container = stack_pointer[-3]; v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); int err; if (slice == NULL) { err = 1; } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); @@ -872,7 +906,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -914,7 +948,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -953,7 +987,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -976,9 +1010,13 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); @@ -1018,7 +1056,6 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); break; } @@ -1031,14 +1068,12 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - stack_pointer[0].bits = (uintptr_t)new_frame; - stack_pointer += 1; + stack_pointer[-2].bits = (uintptr_t)new_frame; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1049,8 +1084,9 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)) < 0) JUMP_TO_ERROR(); + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1062,8 +1098,10 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1079,7 +1117,9 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); @@ -1122,7 +1162,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -1142,9 +1182,11 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(dict_st); if (err) JUMP_TO_ERROR(); stack_pointer += -3; @@ -1158,8 +1200,10 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) JUMP_TO_ERROR(); @@ -1174,7 +1218,9 @@ oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); @@ -1192,7 +1238,9 @@ assert(oparg <= MAX_INTRINSIC_2); PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) JUMP_TO_ERROR(); @@ -1210,6 +1258,7 @@ #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1219,9 +1268,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -1241,22 +1290,30 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) JUMP_TO_ERROR(); if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); if (true) JUMP_TO_ERROR(); } @@ -1269,7 +1326,9 @@ _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { JUMP_TO_ERROR(); } @@ -1285,7 +1344,9 @@ _PyStackRef iter; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) JUMP_TO_ERROR(); iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -1340,6 +1401,7 @@ assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1353,15 +1415,15 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - LOAD_SP(); - value = retval; + value = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; @@ -1373,9 +1435,11 @@ _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_XSETREF(exc_info->exc_value, PyStackRef_Is(exc_value, PyStackRef_None) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1385,16 +1449,16 @@ _PyStackRef value; oparg = CURRENT_OPARG(); // Keep in sync with _common_constants in opcode.py - switch(oparg) { - case CONSTANT_ASSERTIONERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); - break; - case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); - break; - default: - Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; } + value = PyStackRef_FromPyObjectImmortal(val); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1404,10 +1468,15 @@ case _LOAD_BUILD_CLASS: { _PyStackRef bc; PyObject *bc_o; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) JUMP_TO_ERROR(); if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) JUMP_TO_ERROR(); } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -1425,15 +1494,23 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (true) JUMP_TO_ERROR(); } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - else - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1447,16 +1524,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } break; @@ -1469,7 +1552,9 @@ seq = stack_pointer[-1]; output = &stack_pointer[-1]; _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; @@ -1495,10 +1580,10 @@ } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - stack_pointer[0] = val0; val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1563,7 +1648,9 @@ seq = stack_pointer[-1]; right = &stack_pointer[(oparg & 0xFF)]; _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += (oparg & 0xFF) + (oparg >> 8); @@ -1578,8 +1665,10 @@ owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) JUMP_TO_ERROR(); @@ -1593,7 +1682,9 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1606,7 +1697,9 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1617,14 +1710,18 @@ case _DELETE_GLOBAL: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { JUMP_TO_ERROR(); } if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } break; @@ -1634,8 +1731,10 @@ _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) JUMP_TO_ERROR(); } locals = PyStackRef_FromPyObjectNew(l); @@ -1651,7 +1750,9 @@ _PyStackRef v; oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) JUMP_TO_ERROR(); v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -1666,7 +1767,9 @@ oparg = CURRENT_OPARG(); res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR(); null = PyStackRef_NULL; if (oparg & 1) stack_pointer[1] = null; @@ -1690,7 +1793,28 @@ break; } - case _GUARD_BUILTINS_VERSION: { + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { + PyDictKeysObject *globals_keys; + uint16_t version = (uint16_t)CURRENT_OPERAND(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + globals_keys = dict->ma_keys; + assert(DK_IS_UNICODE(globals_keys)); + stack_pointer[0].bits = (uintptr_t)globals_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { + PyDictKeysObject *builtins_keys; uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { @@ -1701,18 +1825,25 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(dict->ma_keys)); + builtins_keys = dict->ma_keys; + assert(DK_IS_UNICODE(builtins_keys)); + stack_pointer[0].bits = (uintptr_t)builtins_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_MODULE: { + case _LOAD_GLOBAL_MODULE_FROM_KEYS: { + PyDictKeysObject *globals_keys; _PyStackRef res; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); + globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = entries[index].me_value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1728,14 +1859,17 @@ break; } - case _LOAD_GLOBAL_BUILTINS: { + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { + PyDictKeysObject *builtins_keys; _PyStackRef res; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); + builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = entries[index].me_value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1755,10 +1889,12 @@ oparg = CURRENT_OPARG(); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) JUMP_TO_ERROR(); } SETLOCAL(oparg, PyStackRef_NULL); @@ -1785,7 +1921,9 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } Py_DECREF(oldobj); @@ -1803,7 +1941,9 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_ERROR(); } @@ -1811,7 +1951,9 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } } @@ -1827,7 +1969,9 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *value_o = PyCell_GetRef(cell); if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) JUMP_TO_ERROR(); } value = PyStackRef_FromPyObjectSteal(value_o); @@ -1842,7 +1986,9 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1852,8 +1998,9 @@ oparg = CURRENT_OPARG(); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { @@ -1924,16 +2071,22 @@ list_st = stack_pointer[-2 - (oparg-1)]; PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); if (true) JUMP_TO_ERROR(); @@ -1951,8 +2104,10 @@ oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1965,7 +2120,9 @@ _PyStackRef set; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -1975,7 +2132,9 @@ int err = 0; for (int i = 0; i < oparg; i++) { if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(values[i]); } @@ -2002,10 +2161,12 @@ } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( values_o, 2, values_o+1, 2, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -2019,20 +2180,28 @@ } case _SETUP_ANNOTATIONS: { - int err; PyObject *ann_dict; if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) JUMP_TO_ERROR(); if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); if (err) JUMP_TO_ERROR(); } @@ -2050,13 +2219,19 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); if (true) JUMP_TO_ERROR(); @@ -2078,9 +2253,13 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); if (true) JUMP_TO_ERROR(); } @@ -2102,11 +2281,13 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) JUMP_TO_ERROR(); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2138,7 +2319,9 @@ } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); @@ -2176,8 +2359,10 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); if (attr_o == NULL) { @@ -2209,7 +2394,9 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. @@ -2232,9 +2419,13 @@ } else { /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) JUMP_TO_ERROR(); + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; @@ -2440,8 +2631,8 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); - stack_pointer[-1] = attr; PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; break; } @@ -2462,8 +2653,8 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); - stack_pointer[-1] = attr; PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2499,9 +2690,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; break; } @@ -2515,9 +2706,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2553,8 +2744,7 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; stack_pointer[-1].bits = (uintptr_t)new_frame; break; @@ -2626,7 +2816,6 @@ JUMP_TO_JUMP_TARGET(); } PyObject *old_value; - uint64_t new_version; if (!DK_IS_UNICODE(dict->ma_keys)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2642,9 +2831,10 @@ } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; - new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); ep->me_value = PyStackRef_AsPyObjectSteal(value); - dict->ma_version_tag = new_version; // PEP 509 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -2683,21 +2873,29 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) JUMP_TO_ERROR(); if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); if (res_bool < 0) JUMP_TO_ERROR(); res = res_bool ? PyStackRef_True : PyStackRef_False; } else { res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2716,8 +2914,8 @@ double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; @@ -2750,8 +2948,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; @@ -2772,8 +2970,8 @@ STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); @@ -2817,7 +3015,9 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) JUMP_TO_ERROR(); @@ -2843,7 +3043,9 @@ } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) JUMP_TO_ERROR(); @@ -2868,7 +3070,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) JUMP_TO_ERROR(); @@ -2888,7 +3092,9 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); @@ -2896,15 +3102,23 @@ } PyObject *match_o = NULL; PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) JUMP_TO_ERROR(); assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) JUMP_TO_ERROR(); if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -2922,12 +3136,16 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(right); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; @@ -2942,9 +3160,11 @@ fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) JUMP_TO_ERROR(); @@ -2961,7 +3181,9 @@ oparg = CURRENT_OPARG(); from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -2994,7 +3216,9 @@ _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) JUMP_TO_ERROR(); PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) JUMP_TO_ERROR(); @@ -3017,10 +3241,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(subject); PyStackRef_CLOSE(type); PyStackRef_CLOSE(names); @@ -3070,8 +3296,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) JUMP_TO_ERROR(); values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -3085,9 +3313,12 @@ _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (PyStackRef_IsNull(iter)) JUMP_TO_ERROR(); + if (iter_o == NULL) JUMP_TO_ERROR(); + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } @@ -3103,23 +3334,29 @@ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } iter = iterable; } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } else { - /* `iterable` is not a generator. */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); - if (PyStackRef_IsNull(iter)) { - JUMP_TO_ERROR(); + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(iter)) { + JUMP_TO_ERROR(); + } + PyStackRef_CLOSE(iterable); } - PyStackRef_CLOSE(iterable); } stack_pointer[-1] = iter; break; @@ -3133,15 +3370,21 @@ iter = stack_pointer[-1]; /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ @@ -3340,19 +3583,27 @@ PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; - attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); - if (PyStackRef_IsNull(attr)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } + if (true) JUMP_TO_ERROR(); } - if (PyStackRef_IsNull(attr)) JUMP_TO_ERROR(); - self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3392,9 +3643,12 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); - res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); - if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3402,9 +3656,10 @@ } case _PUSH_EXC_INFO: { - _PyStackRef new_exc; + _PyStackRef exc; _PyStackRef prev_exc; - new_exc = stack_pointer[-1]; + _PyStackRef new_exc; + exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); @@ -3412,8 +3667,9 @@ else { prev_exc = PyStackRef_None; } - assert(PyStackRef_ExceptionInstanceCheck(new_exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -3459,8 +3715,8 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3480,8 +3736,8 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3545,8 +3801,8 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3556,25 +3812,23 @@ case _MAYBE_EXPAND_METHOD: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; - _PyStackRef func; + _PyStackRef *callable; + _PyStackRef *func; _PyStackRef *maybe_self; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; - func = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); - } - else { - func = callable; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } break; } @@ -3586,13 +3840,13 @@ case _PY_FRAME_GENERAL: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -3602,17 +3856,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, args, total_args, NULL, frame ); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { JUMP_TO_ERROR(); } + new_frame = temp; stack_pointer[0].bits = (uintptr_t)new_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3620,11 +3876,11 @@ } case _CHECK_FUNCTION_VERSION: { - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3639,12 +3895,12 @@ case _CHECK_METHOD_VERSION: { _PyStackRef *null; - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3667,29 +3923,30 @@ case _EXPAND_METHOD: { _PyStackRef *null; - _PyStackRef callable; - _PyStackRef method; + _PyStackRef *callable; + _PyStackRef *method; _PyStackRef *self; oparg = CURRENT_OPARG(); null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; + method = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = method; - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); break; } case _CHECK_IS_NOT_PY_CALLABLE: { - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3704,16 +3961,16 @@ case _CALL_NON_PY_GENERAL: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -3722,20 +3979,22 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -3749,15 +4008,15 @@ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { _PyStackRef *null; - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3765,18 +4024,21 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _PyStackRef callable; - _PyStackRef func; + _PyStackRef *null; + _PyStackRef *callable; + _PyStackRef *func; _PyStackRef *self; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); break; } @@ -3790,11 +4052,11 @@ case _CHECK_FUNCTION_EXACT_ARGS: { _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -3806,10 +4068,10 @@ } case _CHECK_STACK_SPACE: { - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { @@ -3826,18 +4088,16 @@ case _INIT_CALL_PY_EXACT_ARGS_0: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3852,18 +4112,16 @@ case _INIT_CALL_PY_EXACT_ARGS_1: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3878,18 +4136,16 @@ case _INIT_CALL_PY_EXACT_ARGS_2: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3904,18 +4160,16 @@ case _INIT_CALL_PY_EXACT_ARGS_3: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3930,18 +4184,16 @@ case _INIT_CALL_PY_EXACT_ARGS_4: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3956,17 +4208,15 @@ case _INIT_CALL_PY_EXACT_ARGS: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3984,12 +4234,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -4047,7 +4298,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); stack_pointer[-3] = res; @@ -4077,7 +4330,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); stack_pointer[-3] = res; @@ -4088,17 +4343,19 @@ case _CHECK_AND_ALLOCATE_OBJECT: { _PyStackRef *args; - _PyStackRef null; - _PyStackRef callable; - _PyStackRef self; - _PyStackRef init; + _PyStackRef *null; + _PyStackRef *callable; + _PyStackRef *init; + _PyStackRef *self; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4120,36 +4377,39 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); - if (PyStackRef_IsNull(self)) { + PyObject *self_o = _PyType_NewManagedObject(tp); + if (self_o == NULL) { JUMP_TO_ERROR(); } - PyStackRef_CLOSE(callable); - init = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-1 - oparg] = init; - stack_pointer[-2 - oparg] = self; + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); break; } case _CREATE_INIT_FRAME: { _PyStackRef *args; - _PyStackRef init; - _PyStackRef self; + _PyStackRef *self; + _PyStackRef *init; _PyInterpreterFrame *init_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - init = stack_pointer[-1 - oparg]; - self = stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self); - PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); - args[-1] = self; + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); init_frame = _PyEvalFramePushAndInit( - tstate, init_func, NULL, args-1, oparg+1, NULL, shim); - stack_pointer += -2 - oparg; + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); if (init_frame == NULL) { _PyEval_FrameClearAndPop(tstate, shim); @@ -4160,9 +4420,6 @@ * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; - stack_pointer[0].bits = (uintptr_t)init_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -4171,9 +4428,11 @@ should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; @@ -4184,13 +4443,13 @@ case _CALL_BUILTIN_CLASS: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4208,20 +4467,22 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4233,14 +4494,14 @@ case _CALL_BUILTIN_O: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4267,11 +4528,13 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4283,14 +4546,14 @@ case _CALL_BUILTIN_FAST: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4309,24 +4572,26 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable_o), args_o, total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4338,14 +4603,14 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4361,26 +4626,30 @@ } STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4392,14 +4661,14 @@ case _CALL_LEN: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4417,7 +4686,9 @@ STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { JUMP_TO_ERROR(); } @@ -4426,7 +4697,7 @@ if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4438,14 +4709,14 @@ case _CALL_ISINSTANCE: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4463,7 +4734,9 @@ STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { JUMP_TO_ERROR(); } @@ -4471,7 +4744,7 @@ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(inst_stackref); PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4518,13 +4791,13 @@ case _CALL_METHOD_DESCRIPTOR_O: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4559,14 +4832,16 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyStackRef_AsPyObjectBorrow(self_stackref), PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4578,13 +4853,13 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4608,25 +4883,27 @@ } STAT_INC(CALL, hit); int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4638,14 +4915,14 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4679,11 +4956,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4695,13 +4974,13 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4724,26 +5003,28 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } if (true) JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -4754,20 +5035,49 @@ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ + case _MAYBE_EXPAND_METHOD_KW: { + _PyStackRef kwnames_in; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + oparg = CURRENT_OPARG(); + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + stack_pointer[-1] = kwnames_out; + break; + } + /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _PY_FRAME_KW: { _PyStackRef kwnames; _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -4779,30 +5089,30 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { JUMP_TO_ERROR(); } - stack_pointer[0].bits = (uintptr_t)new_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_FUNCTION_VERSION_KW: { - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-3 - oparg]; + callable = &stack_pointer[-3 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4817,12 +5127,12 @@ case _CHECK_METHOD_VERSION_KW: { _PyStackRef *null; - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + callable = &stack_pointer[-3 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4844,33 +5154,31 @@ } case _EXPAND_METHOD_KW: { - _PyStackRef kwnames; _PyStackRef *null; - _PyStackRef callable; - _PyStackRef method; + _PyStackRef *callable; + _PyStackRef *method; _PyStackRef *self; oparg = CURRENT_OPARG(); - kwnames = stack_pointer[-1]; null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + callable = &stack_pointer[-3 - oparg]; + method = &stack_pointer[-3 - oparg]; self = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-3 - oparg] = method; - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); - stack_pointer[-1] = kwnames; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); break; } case _CHECK_IS_NOT_PY_CALLABLE_KW: { - _PyStackRef callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4886,17 +5194,17 @@ _PyStackRef kwnames; _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef res; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + callable = &stack_pointer[-3 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -4905,7 +5213,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -4915,17 +5223,19 @@ } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3 - oparg] = res; @@ -4936,19 +5246,55 @@ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ - /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _MAKE_CALLARGS_A_TUPLE: { + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef callargs; + _PyStackRef func; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_ERROR(); + } + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } + kwargs_out = kwargs_in; + stack_pointer[-1 - (oparg & 1)] = tuple; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; + break; + } + + /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAKE_FUNCTION: { _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) { - JUMP_TO_ERROR(); - } + if (func_obj == NULL) JUMP_TO_ERROR(); _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); @@ -4957,43 +5303,22 @@ } case _SET_FUNCTION_ATTRIBUTE: { - _PyStackRef func_st; + _PyStackRef func_in; _PyStackRef attr_st; + _PyStackRef func_out; oparg = CURRENT_OPARG(); - func_st = stack_pointer[-1]; + func_in = stack_pointer[-1]; attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - case MAKE_FUNCTION_ANNOTATE: - assert(PyCallable_Check(attr)); - assert(func_obj->func_annotate == NULL); - func_obj->func_annotate = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func_st; + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -5001,12 +5326,12 @@ case _RETURN_GENERATOR: { _PyStackRef res; - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - JUMP_TO_ERROR(); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) JUMP_TO_ERROR(); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -5016,12 +5341,12 @@ gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -5061,7 +5386,9 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (result_o == NULL) JUMP_TO_ERROR(); result = PyStackRef_FromPyObjectSteal(result_o); @@ -5077,9 +5404,12 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value_o)) { - res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; @@ -5094,7 +5424,9 @@ _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) JUMP_TO_ERROR(); @@ -5128,7 +5460,9 @@ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) JUMP_TO_ERROR(); @@ -5140,14 +5474,18 @@ } case _SWAP: { - _PyStackRef top; - _PyStackRef bottom; + _PyStackRef top_in; + _PyStackRef bottom_in; + _PyStackRef top_out; + _PyStackRef bottom_out; oparg = CURRENT_OPARG(); - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; break; } @@ -5170,54 +5508,58 @@ case _GUARD_IS_TRUE_POP: { _PyStackRef flag; flag = stack_pointer[-1]; + int is_true = PyStackRef_Is(flag, PyStackRef_True); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (!PyStackRef_Is(flag, PyStackRef_True)) { + if (!is_true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyStackRef_Is(flag, PyStackRef_True)); break; } case _GUARD_IS_FALSE_POP: { _PyStackRef flag; flag = stack_pointer[-1]; + int is_false = PyStackRef_Is(flag, PyStackRef_False); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (!PyStackRef_Is(flag, PyStackRef_False)) { + if (!is_false) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyStackRef_Is(flag, PyStackRef_False)); break; } case _GUARD_IS_NONE_POP: { _PyStackRef val; val = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - if (!PyStackRef_Is(val, PyStackRef_None)) { + int is_none = PyStackRef_Is(val, PyStackRef_None); + if (!is_none) { PyStackRef_CLOSE(val); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); if (1) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { _PyStackRef val; val = stack_pointer[-1]; + int is_none = PyStackRef_Is(val, PyStackRef_None); + PyStackRef_CLOSE(val); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (PyStackRef_Is(val, PyStackRef_None)) { + if (is_none) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyStackRef_CLOSE(val); break; } @@ -5265,12 +5607,14 @@ #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); } #endif if (exit->executor && !exit->executor->vm_data.valid) { @@ -5291,7 +5635,9 @@ } else { int chain_depth = current_executor->vm_data.chain_depth + 1; + _PyFrame_SetStackPointer(frame, stack_pointer); int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { @@ -5353,8 +5699,8 @@ _PyStackRef null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); value = PyStackRef_FromPyObjectNew(ptr); - stack_pointer[0] = value; null = PyStackRef_NULL; + stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); @@ -5376,14 +5722,59 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } break; } + case _LOAD_GLOBAL_MODULE: { + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { _PyStackRef opt; opt = stack_pointer[-1]; @@ -5402,12 +5793,14 @@ #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); } #endif _PyExecutorObject *executor; @@ -5421,7 +5814,9 @@ exit->temperature = advance_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } + _PyFrame_SetStackPointer(frame, stack_pointer); int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { @@ -5448,6 +5843,15 @@ break; } + case _MAKE_WARM: { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + break; + } + case _FATAL_ERROR: { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index f7d8efb28e21c4..388862912d6826 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1001,13 +1001,14 @@ remove_unreachable(basicblock *entryblock) { basicblock **sp = stack; entryblock->b_predecessors = 1; *sp++ = entryblock; + entryblock->b_visited = 1; while (sp > stack) { basicblock *b = *(--sp); - b->b_visited = 1; if (b->b_next && BB_HAS_FALLTHROUGH(b)) { if (!b->b_next->b_visited) { assert(b->b_next->b_predecessors == 0); *sp++ = b->b_next; + b->b_next->b_visited = 1; } b->b_next->b_predecessors++; } @@ -1017,8 +1018,8 @@ remove_unreachable(basicblock *entryblock) { if (is_jump(instr) || is_block_push(instr)) { target = instr->i_target; if (!target->b_visited) { - assert(target->b_predecessors == 0 || target == b->b_next); *sp++ = target; + target->b_visited = 1; } target->b_predecessors++; } @@ -1589,6 +1590,8 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * switch(nextop) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: + case JUMP_IF_FALSE: + case JUMP_IF_TRUE: { /* Remove LOAD_CONST const; conditional jump */ PyObject* cnt = get_const_value(opcode, oparg, consts); @@ -1600,8 +1603,11 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * if (is_true == -1) { return ERROR; } - INSTR_SET_OP0(inst, NOP); - int jump_if_true = nextop == POP_JUMP_IF_TRUE; + if (PyCompile_OpcodeStackEffect(nextop, 0) == -1) { + /* POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE */ + INSTR_SET_OP0(inst, NOP); + } + int jump_if_true = (nextop == POP_JUMP_IF_TRUE || nextop == JUMP_IF_TRUE); if (is_true == jump_if_true) { bb->b_instr[i+1].i_opcode = JUMP; } @@ -1761,6 +1767,36 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) i -= jump_thread(bb, inst, target, POP_JUMP_IF_TRUE); } break; + case JUMP_IF_FALSE: + switch (target->i_opcode) { + case JUMP: + case JUMP_IF_FALSE: + i -= jump_thread(bb, inst, target, JUMP_IF_FALSE); + continue; + case JUMP_IF_TRUE: + // No need to check for loops here, a block's b_next + // cannot point to itself. + assert(inst->i_target != inst->i_target->b_next); + inst->i_target = inst->i_target->b_next; + i--; + continue; + } + break; + case JUMP_IF_TRUE: + switch (target->i_opcode) { + case JUMP: + case JUMP_IF_TRUE: + i -= jump_thread(bb, inst, target, JUMP_IF_TRUE); + continue; + case JUMP_IF_FALSE: + // No need to check for loops here, a block's b_next + // cannot point to itself. + assert(inst->i_target != inst->i_target->b_next); + inst->i_target = inst->i_target->b_next; + i--; + continue; + } + break; case JUMP: case JUMP_NO_INTERRUPT: switch (target->i_opcode) { @@ -2367,6 +2403,38 @@ push_cold_blocks_to_end(cfg_builder *g) { return SUCCESS; } +static int +convert_pseudo_conditional_jumps(cfg_builder *g) +{ + basicblock *entryblock = g->g_entryblock; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (instr->i_opcode == JUMP_IF_FALSE || instr->i_opcode == JUMP_IF_TRUE) { + assert(i == b->b_iused - 1); + instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? + POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; + location loc = instr->i_loc; + cfg_instr copy = { + .i_opcode = COPY, + .i_oparg = 1, + .i_loc = loc, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, ©)); + cfg_instr to_bool = { + .i_opcode = TO_BOOL, + .i_oparg = 0, + .i_loc = loc, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); + } + } + } + return SUCCESS; +} + static int convert_pseudo_ops(cfg_builder *g) { @@ -2826,6 +2894,8 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, int *stackdepth, int *nlocalsplus, _PyInstructionSequence *seq) { + RETURN_IF_ERROR(convert_pseudo_conditional_jumps(g)); + *stackdepth = calculate_stackdepth(g); if (*stackdepth < 0) { return ERROR; diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index ebd67214f43042..16f711184990ac 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -740,7 +740,7 @@ get_locale_info(enum LocaleType type, LocaleInfo *locale_info) break; case LT_NO_LOCALE: locale_info->decimal_point = PyUnicode_FromOrdinal('.'); - locale_info->thousands_sep = PyUnicode_New(0, 0); + locale_info->thousands_sep = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (!locale_info->decimal_point || !locale_info->thousands_sep) return -1; locale_info->grouping = no_grouping; diff --git a/Python/frame.c b/Python/frame.c index d7bb29811bfa50..35e6c2d0a93333 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -13,11 +13,8 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) { Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); - Py_VISIT(frame->f_funcobj); - int err = _PyGC_VisitStackRef(&frame->f_executable, visit, arg); - if (err) { - return err; - } + _Py_VISIT_STACKREF(frame->f_funcobj); + _Py_VISIT_STACKREF(frame->f_executable); return _PyGC_VisitFrameStack(frame, visit, arg); } @@ -126,7 +123,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_DECREF(f); } _PyFrame_ClearLocals(frame); - Py_DECREF(frame->f_funcobj); + PyStackRef_CLEAR(frame->f_funcobj); } /* Unstable API functions */ diff --git a/Python/gc.c b/Python/gc.c index 024d041437be4a..028657eb8999c1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1944,6 +1944,13 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } } +static void +finalize_unlink_gc_head(PyGC_Head *gc) { + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); +} void _PyGC_Fini(PyInterpreterState *interp) @@ -1952,9 +1959,25 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ + /* Prevent a subtle bug that affects sub-interpreters that use basic + * single-phase init extensions (m_size == -1). Those extensions cause objects + * to be shared between interpreters, via the PyDict_Update(mdict, m_copy) call + * in import_find_extension(). + * + * If they are GC objects, their GC head next or prev links could refer to + * the interpreter _gc_runtime_state PyGC_Head nodes. Those nodes go away + * when the interpreter structure is freed and so pointers to them become + * invalid. If those objects are still used by another interpreter and + * UNTRACK is called on them, a crash will happen. We untrack the nodes + * here to avoid that. + * + * This bug was originally fixed when reported as gh-90228. The bug was + * re-introduced in gh-94673. + */ + finalize_unlink_gc_head(&gcstate->young.head); + finalize_unlink_gc_head(&gcstate->old[0].head); + finalize_unlink_gc_head(&gcstate->old[1].head); + finalize_unlink_gc_head(&gcstate->permanent_generation.head); } /* for debugging */ diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 3e003b0280940a..3814cd46e2b440 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -15,7 +15,7 @@ #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" -#include "pycore_typeid.h" // _PyType_MergeThreadLocalRefcounts +#include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts #ifdef Py_GIL_DISABLED @@ -200,6 +200,7 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) } } + frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj); for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { *ref = PyStackRef_AsStrongReference(*ref); @@ -216,12 +217,12 @@ disable_deferred_refcounting(PyObject *op) merge_refcount(op, 0); } - // Heap types also use thread-local refcounting -- disable it here. + // Heap types also use per-thread refcounting -- disable it here. if (PyType_Check(op)) { - // Disable thread-local refcounting for heap types - PyTypeObject *type = (PyTypeObject *)op; - if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - _PyType_ReleaseId((PyHeapTypeObject *)op); + if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) { + PyHeapTypeObject *ht = (PyHeapTypeObject *)op; + _PyObject_ReleaseUniqueId(ht->unique_id); + ht->unique_id = -1; } } @@ -1000,9 +1001,7 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); /* locals and stack */ for (; ref < frame->stackpointer; ref++) { - if (_PyGC_VisitStackRef(ref, visit, arg) < 0) { - return -1; - } + _Py_VISIT_STACKREF(*ref); } return 0; } @@ -1228,7 +1227,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; // merge per-thread refcount for types into the type's actual refcount - _PyType_MergeThreadLocalRefcounts(tstate); + _PyObject_MergePerThreadRefcounts(tstate); // merge refcounts for all queued objects merge_queued_objects(tstate, state); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6d902e2c1d9ba8..a9290986c24f45 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -14,21 +14,23 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -42,7 +44,9 @@ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) goto pop_2_error; @@ -63,9 +67,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); @@ -80,8 +84,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -99,9 +103,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); @@ -114,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -134,9 +138,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); @@ -149,8 +153,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -168,9 +172,9 @@ _PyStackRef left; _PyStackRef right; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); @@ -179,7 +183,9 @@ /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE { + #ifndef NDEBUG PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + #endif PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); int next_oparg; #if TIER_ONE @@ -203,11 +209,11 @@ * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - _Py_DECREF_NO_DEALLOC(left_o); + PyStackRef_CLOSE(left); PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) goto pop_2_error; #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -230,9 +236,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); @@ -247,8 +253,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -266,9 +272,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); @@ -281,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -301,9 +307,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); @@ -318,8 +324,8 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o; - DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -337,9 +343,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); @@ -352,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -379,12 +385,14 @@ #endif /* ENABLE_SPECIALIZATION */ } // _BINARY_SLICE - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -392,8 +400,14 @@ res_o = NULL; } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(container); if (res_o == NULL) goto pop_3_error; @@ -410,21 +424,24 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; _PyStackRef sub; _PyStackRef res; // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; + assert(frame->stackpointer == NULL); #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_SUBSCR); @@ -435,7 +452,9 @@ { PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) goto pop_2_error; @@ -463,9 +482,13 @@ DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); @@ -492,8 +515,8 @@ DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC - container = stack_pointer[-2]; { + container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -506,17 +529,14 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); } // _BINARY_SUBSCR_INIT_CALL - sub = stack_pointer[-1]; { + sub = stack_pointer[-1]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -526,10 +546,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -561,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -593,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -625,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -672,10 +695,12 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( values_o, 2, values_o+1, 2, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -699,7 +724,9 @@ _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -713,7 +740,9 @@ int err = 0; for (int i = 0; i < oparg; i++) { if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(values[i]); } @@ -831,24 +860,26 @@ next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; - _PyStackRef func; + _PyStackRef *func; _PyStackRef *maybe_self; _PyStackRef res; // _SPECIALIZE_CALL - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -857,27 +888,26 @@ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD - args = &stack_pointer[-oparg]; { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; - func = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); - } - else { - func = callable; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } } // _DO_CALL - self_or_null = maybe_self; - callable = func; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -891,12 +921,15 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, total_args, NULL, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -908,7 +941,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -918,30 +951,36 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(res_o); } } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -955,15 +994,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -973,15 +1014,15 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; + _PyStackRef *callable; + _PyStackRef *null; _PyStackRef *args; - _PyStackRef self; - _PyStackRef init; + _PyStackRef *init; + _PyStackRef *self; _PyInterpreterFrame *init_frame; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -990,13 +1031,15 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_AND_ALLOCATE_OBJECT - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != type_version, CALL); @@ -1006,26 +1049,33 @@ PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); - self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp)); - if (PyStackRef_IsNull(self)) { + PyObject *self_o = _PyType_NewManagedObject(tp); + if (self_o == NULL) { goto error; } - PyStackRef_CLOSE(callable); - init = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-1 - oparg] = init; + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); } // _CREATE_INIT_FRAME { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self); - PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); - args[-1] = self; + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); init_frame = _PyEvalFramePushAndInit( - tstate, init_func, NULL, args-1, oparg+1, NULL, shim); - stack_pointer += -2 - oparg; + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); if (init_frame == NULL) { _PyEval_FrameClearAndPop(tstate, shim); @@ -1038,15 +1088,18 @@ tstate->py_recursion_remaining--; } // _PUSH_FRAME - new_frame = init_frame; { + new_frame = init_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1056,13 +1109,13 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *null; - _PyStackRef func; + _PyStackRef *func; _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1073,36 +1126,37 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { + func = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); } // flush // _CHECK_FUNCTION_VERSION - callable = stack_pointer[-2 - oparg]; { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -1110,20 +1164,18 @@ } // _CHECK_STACK_SPACE { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -1144,12 +1196,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1159,13 +1212,13 @@ } TARGET(CALL_BOUND_METHOD_GENERAL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *null; - _PyStackRef method; + _PyStackRef *method; _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1176,11 +1229,11 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_METHOD_VERSION - null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL); @@ -1189,23 +1242,24 @@ } // _EXPAND_METHOD { + method = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = method; - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); } // flush // _PY_FRAME_GENERAL - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -1215,17 +1269,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, args, total_args, NULL, frame ); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -1241,10 +1297,11 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1258,18 +1315,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_CLASS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1281,7 +1338,7 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1292,13 +1349,15 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1309,15 +1368,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1331,19 +1392,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1356,7 +1417,7 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1367,17 +1428,19 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable_o), args_o, total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1388,15 +1451,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1410,19 +1475,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST_WITH_KEYWORDS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1432,12 +1497,14 @@ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1448,14 +1515,16 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1466,15 +1535,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1488,19 +1559,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_O - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1515,11 +1586,13 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1530,15 +1603,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -1552,58 +1627,91 @@ next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; + _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; _PyStackRef func_st; _PyStackRef callargs_st; _PyStackRef kwargs_st = PyStackRef_NULL; _PyStackRef result; - // __DO_CALL_FUNCTION_EX - if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } - callargs_st = stack_pointer[-1 - (oparg & 1)]; - func_st = stack_pointer[-3 - (oparg & 1)]; + // _MAKE_CALLARGS_A_TUPLE { - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - int err = check_args_iterable(tstate, func, callargs); + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { goto error; } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { goto error; } - PyStackRef_CLOSE(callargs_st); - callargs_st = PyStackRef_FromPyObjectSteal(tuple); - callargs = tuple; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); } + kwargs_out = kwargs_in; + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) goto error; - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (PyStackRef_IsNull(result)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLEAR(result); + Py_CLEAR(result_o); } } } @@ -1616,11 +1724,16 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, nargs, callargs, kwargs, frame); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; } @@ -1628,30 +1741,38 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(func_st); - PyStackRef_CLOSE(callargs_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); - assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); - if (PyStackRef_IsNull(result)) { + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) { stack_pointer += -3 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); goto error; } + result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3 - (oparg & 1)] = result; @@ -1668,7 +1789,9 @@ _PyStackRef res; value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); @@ -1688,7 +1811,9 @@ assert(oparg <= MAX_INTRINSIC_2); PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) goto pop_2_error; @@ -1704,7 +1829,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; @@ -1712,9 +1837,9 @@ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -1726,7 +1851,9 @@ STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { goto error; } @@ -1734,7 +1861,7 @@ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(inst_stackref); PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1746,23 +1873,29 @@ next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 4; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; _PyStackRef res; // _SPECIALIZE_CALL_KW - self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -1770,11 +1903,30 @@ #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + } // _DO_CALL_KW - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -1782,17 +1934,6 @@ args--; total_args++; } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && @@ -1801,13 +1942,17 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -1820,7 +1965,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -1832,23 +1977,30 @@ goto error; } } + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(res_o); } @@ -1856,7 +2008,7 @@ } PyStackRef_CLOSE(kwnames); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -1874,14 +2026,14 @@ } TARGET(CALL_KW_BOUND_METHOD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *null; _PyStackRef kwnames; - _PyStackRef method; + _PyStackRef *method; _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1892,11 +2044,11 @@ DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_METHOD_VERSION_KW - null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL_KW); @@ -1904,26 +2056,26 @@ DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); } // _EXPAND_METHOD_KW - kwnames = stack_pointer[-1]; { + method = &stack_pointer[-3 - oparg]; self = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-3 - oparg] = method; - assert(PyStackRef_FunctionCheck(method)); - PyStackRef_CLOSE(callable); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); } // flush // _PY_FRAME_KW - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -1935,14 +2087,17 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -1962,10 +2117,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1979,7 +2137,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef kwnames; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1987,21 +2145,21 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW - callable = stack_pointer[-3 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); } // _CALL_KW_NON_PY - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2010,7 +2168,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2024,17 +2182,19 @@ } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2045,15 +2205,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3 - oparg] = res; @@ -2063,11 +2225,11 @@ } TARGET(CALL_KW_PY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef kwnames; _PyStackRef *args; @@ -2078,20 +2240,20 @@ DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_FUNCTION_VERSION_KW - callable = stack_pointer[-3 - oparg]; { + callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL_KW); } // _PY_FRAME_KW - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -2103,14 +2265,17 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -2130,10 +2295,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2147,7 +2315,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; @@ -2155,9 +2323,9 @@ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2169,7 +2337,9 @@ STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { goto error; } @@ -2178,7 +2348,7 @@ if (res_o == NULL) { GOTO_ERROR(error); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; @@ -2228,18 +2398,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2253,12 +2423,10 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2269,14 +2437,18 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2287,15 +2459,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2309,18 +2483,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2335,11 +2509,9 @@ DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2350,14 +2522,18 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2368,15 +2544,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2390,19 +2568,19 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_NOARGS - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2421,11 +2599,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2436,15 +2616,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2458,18 +2640,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_O - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2489,14 +2671,16 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyStackRef_AsPyObjectBorrow(self_stackref), PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(self_stackref); PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2507,15 +2691,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2529,27 +2715,27 @@ next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE - callable = stack_pointer[-2 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(PyFunction_Check(callable_o), CALL); DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } // _CALL_NON_PY_GENERAL - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; @@ -2558,7 +2744,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); @@ -2569,13 +2755,15 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -2589,15 +2777,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -2607,11 +2797,11 @@ } TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2621,18 +2811,18 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION - callable = stack_pointer[-2 - oparg]; { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -2640,20 +2830,18 @@ } // _CHECK_STACK_SPACE { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -2674,12 +2862,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2689,11 +2878,11 @@ } TARGET(CALL_PY_GENERAL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2703,19 +2892,19 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION - callable = stack_pointer[-2 - oparg]; { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _PY_FRAME_GENERAL - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -2725,17 +2914,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, args, total_args, NULL, frame ); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -2751,10 +2942,11 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2775,27 +2967,36 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_STR_1 - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null), CALL); DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); if (PyStackRef_IsNull(res)) goto pop_3_error; } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) goto pop_2_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3] = res; @@ -2816,27 +3017,36 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_TUPLE_1 - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null), CALL); DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); if (PyStackRef_IsNull(res)) goto pop_3_error; } // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) goto pop_2_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-3] = res; @@ -2885,7 +3095,9 @@ exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); @@ -2893,15 +3105,23 @@ } PyObject *match_o = NULL; PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) goto pop_2_error; assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) goto pop_2_error; if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -2922,12 +3142,16 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); if (true) goto pop_1_error; } + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(right); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; @@ -2935,7 +3159,7 @@ } TARGET(CLEANUP_THROW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); @@ -2950,21 +3174,25 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - stack_pointer[-2] = value; PyStackRef_CLOSE(sub_iter_st); PyStackRef_CLOSE(last_sent_val_st); PyStackRef_CLOSE(exc_value_st); - none = PyStackRef_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } stack_pointer[-3] = none; + stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -2975,21 +3203,23 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; _PyStackRef right; _PyStackRef res; // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -3001,22 +3231,30 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) goto pop_2_error; if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); - if (res_bool < 0) goto pop_2_error; + if (res_bool < 0) goto error; res = res_bool ? PyStackRef_True : PyStackRef_False; } else { res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3030,9 +3268,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); @@ -3048,8 +3286,8 @@ double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } @@ -3068,9 +3306,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); @@ -3090,8 +3328,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } @@ -3110,9 +3348,9 @@ _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); @@ -3126,8 +3364,8 @@ STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); @@ -3145,20 +3383,22 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); PREDICTED(CONTAINS_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; _PyStackRef right; _PyStackRef b; // _SPECIALIZE_CONTAINS_OP - right = stack_pointer[-1]; { + right = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -3166,11 +3406,13 @@ #endif /* ENABLE_SPECIALIZATION */ } // _CONTAINS_OP - left = stack_pointer[-2]; { + left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; @@ -3197,7 +3439,9 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; @@ -3224,7 +3468,9 @@ DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; @@ -3245,7 +3491,9 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (result_o == NULL) goto pop_1_error; result = PyStackRef_FromPyObjectSteal(result_o); @@ -3274,8 +3522,9 @@ INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { @@ -3292,7 +3541,9 @@ _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) goto pop_1_error; stack_pointer += -1; @@ -3309,7 +3560,9 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } Py_DECREF(oldobj); @@ -3322,10 +3575,12 @@ INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) goto error; } SETLOCAL(oparg, PyStackRef_NULL); @@ -3337,14 +3592,18 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { goto error; } if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } DISPATCH(); @@ -3358,16 +3617,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } DISPATCH(); @@ -3382,8 +3647,10 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) goto pop_2_error; @@ -3405,9 +3672,13 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); if (true) goto pop_1_error; } @@ -3427,13 +3698,19 @@ dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); if (true) goto pop_1_error; @@ -3445,7 +3722,7 @@ } TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); @@ -3455,14 +3732,19 @@ awaitable_st = stack_pointer[-2]; PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { PyStackRef_CLOSE(awaitable_st); PyStackRef_CLOSE(exc_st); } else { Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } stack_pointer += -2; @@ -3488,18 +3770,20 @@ INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; (void)receiver; + val = value; PyStackRef_CLOSE(receiver); - stack_pointer[-2] = value; + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(ENTER_EXECUTOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); @@ -3539,9 +3823,11 @@ should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } stack_pointer += -1; @@ -3571,9 +3857,12 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value_o)) { - res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (PyStackRef_IsNull(res)) goto pop_1_error; + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; @@ -3591,7 +3880,9 @@ _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) goto pop_2_error; @@ -3607,19 +3898,21 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; _PyStackRef next; // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3630,16 +3923,21 @@ { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next_o == NULL) { - next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -3673,8 +3971,8 @@ DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); } // _FOR_ITER_GEN_FRAME - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3689,15 +3987,16 @@ frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); } // _PUSH_FRAME - new_frame = gen_frame; { + new_frame = gen_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -3715,8 +4014,8 @@ _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST @@ -3750,8 +4049,8 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - stack_pointer[0] = next; } + stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3766,8 +4065,8 @@ _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } @@ -3811,8 +4110,8 @@ _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE @@ -3843,8 +4142,8 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - stack_pointer[0] = next; } + stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3865,24 +4164,32 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (true) goto pop_1_error; } + _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) goto pop_1_error; if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); - if (true) goto pop_1_error; + if (true) goto error; } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -3896,7 +4203,9 @@ _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { goto error; } @@ -3914,7 +4223,9 @@ _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) goto pop_1_error; iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -3930,9 +4241,12 @@ _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (PyStackRef_IsNull(iter)) goto pop_1_error; + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); } @@ -3945,7 +4259,9 @@ _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) goto error; PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; @@ -3970,23 +4286,29 @@ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } iter = iterable; } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } else { - /* `iterable` is not a generator. */ - iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); - if (PyStackRef_IsNull(iter)) { - goto error; + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(iter)) { + goto error; + } + PyStackRef_CLOSE(iterable); } - PyStackRef_CLOSE(iterable); } stack_pointer[-1] = iter; DISPATCH(); @@ -4000,7 +4322,9 @@ _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) goto error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -4019,9 +4343,11 @@ fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) goto pop_2_error; @@ -4033,61 +4359,66 @@ } TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - _PyStackRef callable; + _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; - _PyStackRef func; + _PyStackRef *func; _PyStackRef *maybe_self; _PyStackRef res; /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; maybe_self[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; - func = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = func; - PyStackRef_CLOSE(callable); - } - else { - func = callable; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } } // _MONITOR_CALL { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); } - else if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } else { - arg0 = &_PyInstrumentation_MISSING; + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } } + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } // _DO_CALL - self_or_null = maybe_self; - callable = func; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -4101,12 +4432,15 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable[0], locals, args, total_args, NULL, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -4118,7 +4452,7 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -4128,30 +4462,36 @@ goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(res_o); } } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } @@ -4165,15 +4505,17 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); - if (err != 0) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[-2 - oparg] = res; @@ -4190,7 +4532,7 @@ } TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); @@ -4203,16 +4545,18 @@ PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); @@ -4223,7 +4567,9 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } @@ -4235,30 +4581,34 @@ } TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } } + val = value; PyStackRef_CLOSE(receiver); - stack_pointer[-2] = value; + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); @@ -4266,19 +4616,25 @@ _Py_CODEUNIT *target; _PyStackRef iter_stackref = TOP(); PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); target = next_instr; } else { if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -4293,12 +4649,14 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -4310,7 +4668,7 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); @@ -4318,9 +4676,11 @@ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -4332,7 +4692,7 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); @@ -4341,8 +4701,8 @@ } TARGET(INSTRUMENTED_LINE) { - _Py_CODEUNIT *prev_instr = frame->instr_ptr; - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); @@ -4376,7 +4736,7 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); @@ -4385,10 +4745,12 @@ // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); @@ -4405,7 +4767,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); @@ -4428,7 +4790,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); @@ -4451,7 +4813,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); @@ -4468,7 +4830,7 @@ } TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); @@ -4478,7 +4840,9 @@ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } @@ -4493,7 +4857,9 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -4514,7 +4880,7 @@ } TARGET(INSTRUMENTED_RETURN_CONST) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); @@ -4525,22 +4891,29 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); - stack_pointer[0] = value; } // _RETURN_VALUE_EVENT - val = value; { + val = value; + stack_pointer[0] = val; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } // _RETURN_VALUE - retval = val; { + retval = val; #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); @@ -4548,9 +4921,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; @@ -4560,7 +4933,7 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); @@ -4568,19 +4941,22 @@ _PyStackRef retval; _PyStackRef res; // _RETURN_VALUE_EVENT - val = stack_pointer[-1]; { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } // _RETURN_VALUE - retval = val; { + retval = val; #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4590,9 +4966,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; @@ -4602,7 +4978,7 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); @@ -4610,22 +4986,24 @@ _PyStackRef retval; _PyStackRef value; // _YIELD_VALUE_EVENT - val = stack_pointer[-1]; { - SAVE_SP(); + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - LOAD_SP(); - if (err) goto error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; DISPATCH(); } } // _YIELD_VALUE - retval = val; { + retval = val; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -4637,6 +5015,7 @@ assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4650,15 +5029,15 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - LOAD_SP(); - value = retval; + value = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = value; @@ -4680,6 +5059,8 @@ assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } TARGET(IS_OP) { @@ -4708,16 +5089,18 @@ } TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ + QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -4738,7 +5121,9 @@ start--; } _PyExecutorObject *executor; + _PyFrame_SetStackPointer(frame, stack_pointer); int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized < 0) goto error; if (optimized) { assert(tstate->previous_executor == NULL); @@ -4787,8 +5172,9 @@ _PyStackRef v; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)) < 0) goto pop_1_error; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -4804,16 +5190,22 @@ list_st = stack_pointer[-2 - (oparg-1)]; PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); if (true) goto pop_1_error; @@ -4830,21 +5222,23 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; + _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; _PyStackRef attr; _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -4859,7 +5253,9 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. @@ -4882,9 +5278,13 @@ } else { /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) goto pop_1_error; + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -4896,7 +5296,7 @@ } TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -4905,8 +5305,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); @@ -4920,10 +5320,10 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -4931,7 +5331,7 @@ } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -4940,8 +5340,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); @@ -4961,10 +5361,10 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -4972,7 +5372,7 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -4997,8 +5397,8 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -5008,7 +5408,7 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5017,8 +5417,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5053,7 +5453,7 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5062,8 +5462,8 @@ _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5086,9 +5486,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; } + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5096,7 +5496,7 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5105,8 +5505,8 @@ _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5122,9 +5522,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; } + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5132,7 +5532,7 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5141,8 +5541,8 @@ _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5170,9 +5570,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; self = owner; } + stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5180,7 +5580,7 @@ } TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5189,8 +5589,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR); @@ -5223,7 +5623,7 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5231,8 +5631,8 @@ _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5248,13 +5648,13 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; } + stack_pointer[-1] = attr; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5262,8 +5662,8 @@ _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5290,13 +5690,13 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; } + stack_pointer[-1] = attr; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5308,8 +5708,8 @@ DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); } // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5328,8 +5728,7 @@ DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; } // _SAVE_RETURN_OFFSET @@ -5346,12 +5745,13 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -5361,7 +5761,7 @@ } TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5370,8 +5770,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5387,10 +5787,10 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); - stack_pointer[-1] = attr; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -5398,7 +5798,7 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -5407,8 +5807,8 @@ _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -5455,10 +5855,15 @@ INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -5474,16 +5879,16 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; // Keep in sync with _common_constants in opcode.py - switch(oparg) { - case CONSTANT_ASSERTIONERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); - break; - case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); - break; - default: - Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5510,7 +5915,9 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *value_o = PyCell_GetRef(cell); if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } value = PyStackRef_FromPyObjectSteal(value_o); @@ -5554,10 +5961,12 @@ _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (1) goto error; } value = PyStackRef_DUP(value_s); @@ -5597,7 +6006,9 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { goto error; } @@ -5605,7 +6016,9 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } } @@ -5624,23 +6037,30 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - if (err < 0) { - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) goto pop_1_error; if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); } goto error; } @@ -5648,20 +6068,31 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (v_o == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - if (true) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) goto error; } } } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(mod_or_class_dict); v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[-1] = v; DISPATCH(); @@ -5672,7 +6103,7 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *this_instr = next_instr - 5; + _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; _PyStackRef null = PyStackRef_NULL; @@ -5684,7 +6115,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -5698,7 +6131,9 @@ { res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; null = PyStackRef_NULL; } @@ -5709,10 +6144,11 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *builtins_keys; _PyStackRef res; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -5724,19 +6160,19 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } - // _GUARD_BUILTINS_VERSION + // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + builtins_keys = dict->ma_keys; + assert(DK_IS_UNICODE(builtins_keys)); } - // _LOAD_GLOBAL_BUILTINS + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = entries[index].me_value; DEOPT_IF(res_o == NULL, LOAD_GLOBAL); Py_INCREF(res_o); @@ -5752,27 +6188,28 @@ } TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *globals_keys; _PyStackRef res; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION + // _GUARD_GLOBALS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + globals_keys = dict->ma_keys; + assert(DK_IS_UNICODE(globals_keys)); } /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE + // _LOAD_GLOBAL_MODULE_FROM_KEYS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = entries[index].me_value; DEOPT_IF(res_o == NULL, LOAD_GLOBAL); Py_INCREF(res_o); @@ -5794,8 +6231,10 @@ _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } locals = PyStackRef_FromPyObjectNew(l); @@ -5811,7 +6250,9 @@ INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) goto error; v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -5832,19 +6273,27 @@ PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; - attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); - if (PyStackRef_IsNull(attr)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } + if (true) goto error; } - if (PyStackRef_IsNull(attr)) goto pop_1_error; - self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5854,7 +6303,7 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; _PyStackRef class_st; @@ -5862,16 +6311,18 @@ _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -5879,33 +6330,46 @@ #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR - self_st = stack_pointer[-1]; { + self_st = stack_pointer[-1]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - if (err) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (true) goto pop_3_error; + } } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(super); } @@ -5916,14 +6380,19 @@ PyStackRef_CLOSE(self_st); if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(super); - if (PyStackRef_IsNull(attr)) goto pop_3_error; + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); null = PyStackRef_NULL; } - stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; - stack_pointer += -2 + (oparg & 1); + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5949,7 +6418,9 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); @@ -5985,8 +6456,10 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); if (attr_o == NULL) { @@ -6030,12 +6503,12 @@ _PyStackRef func; codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) { - goto error; - } + if (func_obj == NULL) goto pop_1_error; _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); @@ -6057,11 +6530,13 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto pop_2_error; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -6082,10 +6557,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(subject); PyStackRef_CLOSE(type); PyStackRef_CLOSE(names); @@ -6114,8 +6591,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) goto error; values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -6168,16 +6647,18 @@ _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_XSETREF(exc_info->exc_value, PyStackRef_Is(exc_value, PyStackRef_None) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); @@ -6196,7 +6677,7 @@ } TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); @@ -6205,8 +6686,8 @@ _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE - value = stack_pointer[-1]; { + value = stack_pointer[-1]; if (PyStackRef_Is(value, PyStackRef_None)) { b = PyStackRef_True; } @@ -6216,8 +6697,8 @@ } } // _POP_JUMP_IF_TRUE - cond = b; { + cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION @@ -6231,7 +6712,7 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); @@ -6240,8 +6721,8 @@ _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE - value = stack_pointer[-1]; { + value = stack_pointer[-1]; if (PyStackRef_Is(value, PyStackRef_None)) { b = PyStackRef_True; } @@ -6251,8 +6732,8 @@ } } // _POP_JUMP_IF_FALSE - cond = b; { + cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION @@ -6266,7 +6747,7 @@ } TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); @@ -6300,9 +6781,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef new_exc; + _PyStackRef exc; _PyStackRef prev_exc; - new_exc = stack_pointer[-1]; + _PyStackRef new_exc; + exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); @@ -6310,8 +6792,9 @@ else { prev_exc = PyStackRef_None; } - assert(PyStackRef_ExceptionInstanceCheck(new_exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -6332,41 +6815,32 @@ } TARGET(RAISE_VARARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = PyStackRef_AsPyObjectSteal(args[1]); - _Py_FALLTHROUGH; - case 1: - exc = PyStackRef_AsPyObjectSteal(args[0]); - _Py_FALLTHROUGH; - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; - } - if (true) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; } + if (true) goto error; } TARGET(RERAISE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); @@ -6374,7 +6848,7 @@ _PyStackRef exc_st; exc_st = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); @@ -6383,14 +6857,22 @@ assert(!_PyErr_Occurred(tstate)); } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(exc); goto error; } } assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } @@ -6408,7 +6890,7 @@ next_instr += 1; INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; + _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; // _MAYBE_INSTRUMENT { @@ -6416,7 +6898,9 @@ uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; } @@ -6439,7 +6923,9 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; } } @@ -6473,14 +6959,14 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); - stack_pointer[0] = value; } // _RETURN_VALUE - retval = value; { + retval = value; #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); @@ -6488,9 +6974,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; @@ -6504,12 +6990,12 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) goto error; assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -6519,12 +7005,12 @@ gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -6542,6 +7028,7 @@ #if TIER_ONE assert(frame != &entry_frame); #endif + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6551,9 +7038,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - res = retval; + res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; @@ -6566,20 +7053,22 @@ next_instr += 2; INSTRUCTION_STATS(SEND); PREDICTED(SEND); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; _PyStackRef v; _PyStackRef retval; // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; { + receiver = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -6587,8 +7076,8 @@ #endif /* ENABLE_SPECIALIZATION */ } // _SEND - v = stack_pointer[-1]; { + v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; assert(frame != &entry_frame); @@ -6610,25 +7099,36 @@ DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); retval_o = PyObject_CallMethodOneArg(receiver_o, &_Py_ID(send), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); } + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); } else { - goto error; + PyStackRef_CLOSE(v); + if (true) goto pop_1_error; } } PyStackRef_CLOSE(v); @@ -6653,9 +7153,9 @@ DEOPT_IF(tstate->interp->eval_frame, SEND); } // _SEND_GEN_FRAME - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); @@ -6670,17 +7170,18 @@ gen_frame->previous = frame; } // _PUSH_FRAME - new_frame = gen_frame; { + new_frame = gen_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -6693,20 +7194,28 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - int err; PyObject *ann_dict; if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); if (true) goto error; } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); if (err) goto error; } @@ -6724,8 +7233,10 @@ _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -6738,41 +7249,20 @@ next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; - _PyStackRef func_st; - func_st = stack_pointer[-1]; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - case MAKE_FUNCTION_ANNOTATE: - assert(PyCallable_Check(attr)); - assert(func_obj->func_annotate == NULL); - func_obj->func_annotate = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func_st; + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6786,8 +7276,10 @@ _PyStackRef iterable; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; @@ -6800,20 +7292,22 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; + _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -6822,11 +7316,13 @@ } /* Skip 3 cache entries */ // _STORE_ATTR - v = stack_pointer[-2]; { + v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) goto pop_2_error; @@ -6837,7 +7333,7 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -6845,8 +7341,8 @@ _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -6861,8 +7357,8 @@ DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); @@ -6886,7 +7382,7 @@ } TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -6894,16 +7390,16 @@ _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; @@ -6919,7 +7415,7 @@ } TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -6927,16 +7423,16 @@ _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t hint = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); @@ -6946,7 +7442,6 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; - uint64_t new_version; DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); @@ -6956,9 +7451,10 @@ } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; - new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); ep->me_value = PyStackRef_AsPyObjectSteal(value); - dict->ma_version_tag = new_version; // PEP 509 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -6977,7 +7473,9 @@ _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7034,7 +7532,9 @@ _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -7052,15 +7552,23 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (true) goto pop_1_error; } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - else - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -7084,20 +7592,28 @@ #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SLICE - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); int err; if (slice == NULL) { err = 1; } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); @@ -7113,21 +7629,23 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; _PyStackRef sub; _PyStackRef v; // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -7135,10 +7653,12 @@ #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SUBSCR - v = stack_pointer[-3]; { + v = stack_pointer[-3]; /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); @@ -7164,9 +7684,11 @@ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(dict_st); if (err) goto pop_3_error; stack_pointer += -3; @@ -7200,7 +7722,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -7211,13 +7733,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - _PyStackRef bottom; - _PyStackRef top; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; DISPATCH(); } @@ -7226,19 +7752,21 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; { + value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -7248,7 +7776,9 @@ /* Skip 2 cache entries */ // _TO_BOOL { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) goto pop_1_error; res = err ? PyStackRef_True : PyStackRef_False; @@ -7258,7 +7788,7 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -7267,16 +7797,16 @@ _PyStackRef res; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE - value = owner; { + value = owner; PyStackRef_CLOSE(value); res = PyStackRef_True; } @@ -7312,7 +7842,7 @@ DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortalLoose(value_o)); + assert(_Py_IsImmortal(value_o)); res = PyStackRef_False; } else { @@ -7374,7 +7904,7 @@ DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortalLoose(value_o)); + assert(_Py_IsImmortal(value_o)); res = PyStackRef_False; } else { @@ -7393,7 +7923,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); @@ -7408,7 +7940,9 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); @@ -7439,7 +7973,9 @@ seq = stack_pointer[-1]; right = &stack_pointer[(oparg & 0xFF)]; _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; stack_pointer += (oparg & 0xFF) + (oparg >> 8); @@ -7452,19 +7988,21 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; _PyStackRef *output; // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; { + seq = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -7477,7 +8015,9 @@ { output = &stack_pointer[-1]; _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; } @@ -7550,10 +8090,10 @@ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - stack_pointer[0] = val0; val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7597,9 +8137,12 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); - res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); - if (PyStackRef_IsNull(res)) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -7624,6 +8167,7 @@ assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7637,113 +8181,19 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - LOAD_SP(); - value = retval; + value = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - - TARGET(_DO_CALL_FUNCTION_EX) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(_DO_CALL_FUNCTION_EX); - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; - _PyStackRef result; - if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } - callargs_st = stack_pointer[-1 - (oparg & 1)]; - func_st = stack_pointer[-3 - (oparg & 1)]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - int err = check_args_iterable(tstate, func, callargs); - if (err < 0) { - goto error; - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - goto error; - } - PyStackRef_CLOSE(callargs_st); - callargs_st = PyStackRef_FromPyObjectSteal(tuple); - callargs = tuple; - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) goto error; - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (PyStackRef_IsNull(result)) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - PyStackRef_CLEAR(result); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs, frame); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - goto error; - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); - } - PyStackRef_CLOSE(func_st); - PyStackRef_CLOSE(callargs_st); - PyStackRef_XCLOSE(kwargs_st); - assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); - if (PyStackRef_IsNull(result)) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } #undef TIER_ONE diff --git a/Python/getargs.c b/Python/getargs.c index 1b9b39939ae94e..a764343ea9ee3f 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2064,7 +2064,7 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) for (i = 0; i < nkwargs; i++) { PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); assert(PyUnicode_Check(kwname)); - if (_PyUnicode_EQ(kwname, key)) { + if (_PyUnicode_Equal(kwname, key)) { return Py_NewRef(kwstack[i]); } } diff --git a/Python/hamt.c b/Python/hamt.c index a8fbb00b807934..cfd211f4541446 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -349,7 +349,7 @@ hamt_node_find(PyHamtNode *node, #ifdef Py_DEBUG static int hamt_node_dump(PyHamtNode *node, - _PyUnicodeWriter *writer, int level); + PyUnicodeWriter *writer, int level); #endif static PyHamtNode * @@ -444,7 +444,7 @@ hamt_bitindex(uint32_t bitmap, uint32_t bit) #ifdef Py_DEBUG static int -_hamt_dump_ident(_PyUnicodeWriter *writer, int level) +_hamt_dump_ident(PyUnicodeWriter *writer, int level) { /* Write `' ' * level` to the `writer` */ PyObject *str = NULL; @@ -467,7 +467,7 @@ _hamt_dump_ident(_PyUnicodeWriter *writer, int level) goto error; } - ret = _PyUnicodeWriter_WriteStr(writer, res); + ret = PyUnicodeWriter_WriteStr(writer, res); error: Py_XDECREF(res); @@ -476,29 +476,6 @@ _hamt_dump_ident(_PyUnicodeWriter *writer, int level) return ret; } -static int -_hamt_dump_format(_PyUnicodeWriter *writer, const char *format, ...) -{ - /* A convenient helper combining _PyUnicodeWriter_WriteStr and - PyUnicode_FromFormatV. - */ - PyObject* msg; - int ret; - - va_list vargs; - va_start(vargs, format); - msg = PyUnicode_FromFormatV(format, vargs); - va_end(vargs); - - if (msg == NULL) { - return -1; - } - - ret = _PyUnicodeWriter_WriteStr(writer, msg); - Py_DECREF(msg); - return ret; -} - #endif /* Py_DEBUG */ /////////////////////////////////// Bitmap Node @@ -1154,7 +1131,7 @@ hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) #ifdef Py_DEBUG static int hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for Bitmap nodes. */ @@ -1166,8 +1143,8 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, goto error; } - if (_hamt_dump_format(writer, "BitmapNode(size=%zd count=%zd ", - Py_SIZE(node), Py_SIZE(node) / 2)) + if (PyUnicodeWriter_Format(writer, "BitmapNode(size=%zd count=%zd ", + Py_SIZE(node), Py_SIZE(node) / 2) < 0) { goto error; } @@ -1181,7 +1158,9 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, if (tmp2 == NULL) { goto error; } - if (_hamt_dump_format(writer, "bitmap=%S id=%p):\n", tmp2, node)) { + if (PyUnicodeWriter_Format(writer, "bitmap=%S id=%p):\n", + tmp2, node) < 0) + { Py_DECREF(tmp2); goto error; } @@ -1196,7 +1175,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } if (key_or_null == NULL) { - if (_hamt_dump_format(writer, "NULL:\n")) { + if (PyUnicodeWriter_WriteUTF8(writer, "NULL:\n", -1) < 0) { goto error; } @@ -1207,14 +1186,14 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } } else { - if (_hamt_dump_format(writer, "%R: %R", key_or_null, - val_or_node)) + if (PyUnicodeWriter_Format(writer, "%R: %R", + key_or_null, val_or_node) < 0) { goto error; } } - if (_hamt_dump_format(writer, "\n")) { + if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { goto error; } } @@ -1548,7 +1527,7 @@ hamt_node_collision_dealloc(PyHamtNode_Collision *self) #ifdef Py_DEBUG static int hamt_node_collision_dump(PyHamtNode_Collision *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for Collision nodes. */ @@ -1558,8 +1537,8 @@ hamt_node_collision_dump(PyHamtNode_Collision *node, goto error; } - if (_hamt_dump_format(writer, "CollisionNode(size=%zd id=%p):\n", - Py_SIZE(node), node)) + if (PyUnicodeWriter_Format(writer, "CollisionNode(size=%zd id=%p):\n", + Py_SIZE(node), node) < 0) { goto error; } @@ -1572,7 +1551,7 @@ hamt_node_collision_dump(PyHamtNode_Collision *node, goto error; } - if (_hamt_dump_format(writer, "%R: %R\n", key, val)) { + if (PyUnicodeWriter_Format(writer, "%R: %R\n", key, val) < 0) { goto error; } } @@ -1924,7 +1903,7 @@ hamt_node_array_dealloc(PyHamtNode_Array *self) #ifdef Py_DEBUG static int hamt_node_array_dump(PyHamtNode_Array *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for Array nodes. */ @@ -1934,7 +1913,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (_hamt_dump_format(writer, "ArrayNode(id=%p):\n", node)) { + if (PyUnicodeWriter_Format(writer, "ArrayNode(id=%p):\n", node) < 0) { goto error; } @@ -1947,7 +1926,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (_hamt_dump_format(writer, "%zd::\n", i)) { + if (PyUnicodeWriter_Format(writer, "%zd::\n", i) < 0) { goto error; } @@ -1955,7 +1934,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (_hamt_dump_format(writer, "\n")) { + if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { goto error; } } @@ -2071,7 +2050,7 @@ hamt_node_find(PyHamtNode *node, #ifdef Py_DEBUG static int hamt_node_dump(PyHamtNode *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for a node. @@ -2440,22 +2419,24 @@ _PyHamt_New(void) static PyObject * hamt_dump(PyHamtObject *self) { - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } - if (_hamt_dump_format(&writer, "HAMT(len=%zd):\n", self->h_count)) { + if (PyUnicodeWriter_Format(writer, "HAMT(len=%zd):\n", + self->h_count) < 0) { goto error; } - if (hamt_node_dump(self->h_root, &writer, 0)) { + if (hamt_node_dump(self->h_root, writer, 0)) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } #endif /* Py_DEBUG */ diff --git a/Python/import.c b/Python/import.c index 460b1fe225c72e..acf849f14562b9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1051,7 +1051,7 @@ del_cached_def(struct extensions_cache_value *value) However, this decref would be problematic if the module def were dynamically allocated, it were the last ref, and this function were called with an interpreter other than the def's owner. */ - assert(value->def == NULL || _Py_IsImmortalLoose(value->def)); + assert(value->def == NULL || _Py_IsImmortal(value->def)); Py_XDECREF(value->def->m_base.m_copy); value->def->m_base.m_copy = NULL; diff --git a/Python/initconfig.c b/Python/initconfig.c index d93244f7f41084..c142438b02bfd9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -150,7 +150,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), - SPEC(perf_profiling, BOOL, READ_ONLY, NO_SYS), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), SPEC(program_name, WSTR, READ_ONLY, NO_SYS), SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), @@ -3457,6 +3457,9 @@ PyInitConfig_Create(void) void PyInitConfig_Free(PyInitConfig *config) { + if (config == NULL) { + return; + } free(config->err_msg); free(config); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5e51a9c992f6c2..e1e494c31a1120 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -643,8 +643,8 @@ de_instrument(PyCodeObject *code, int i, int event) CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, - adaptive_counter_warmup().as_counter); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } } @@ -719,8 +719,8 @@ instrument(PyCodeObject *code, int i) assert(instrumented); FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { - FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, - adaptive_counter_warmup().as_counter); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); instr[1].counter = adaptive_counter_warmup(); } } @@ -1660,6 +1660,16 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) if (allocate_instrumentation_data(code)) { return -1; } + // If the local monitors are out of date, clear them up + _Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors; + for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { + if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) { + for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) { + local_monitors->tools[j] &= ~(1 << i); + } + } + } + _Py_LocalMonitors all_events = local_union( interp->monitors, code->_co_monitoring->local_monitors); @@ -2004,6 +2014,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent goto done; } + code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id]; + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { @@ -2036,6 +2048,43 @@ _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent return 0; } +int _PyMonitoring_ClearToolId(int tool_id) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_GET(); + + for (int i = 0; i < _PY_MONITORING_EVENTS; i++) { + PyObject *func = _PyMonitoring_RegisterCallback(tool_id, i, NULL); + if (func != NULL) { + Py_DECREF(func); + } + } + + if (_PyMonitoring_SetEvents(tool_id, 0) < 0) { + return -1; + } + + _PyEval_StopTheWorld(interp); + uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT; + if (version == 0) { + PyErr_Format(PyExc_OverflowError, "events set too many times"); + _PyEval_StartTheWorld(interp); + return -1; + } + + // monitoring_tool_versions[tool_id] is set to latest global version here to + // 1. invalidate local events on all existing code objects + // 2. be ready for the next call to set local events + interp->monitoring_tool_versions[tool_id] = version; + + // Set the new global version so all the code objects can refresh the + // instrumentation. + set_global_version(_PyThreadState_GET(), version); + int res = instrument_all_executing_code_objects(interp); + _PyEval_StartTheWorld(interp); + return res; +} + /*[clinic input] module monitoring [clinic start generated code]*/ @@ -2083,6 +2132,33 @@ monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) Py_RETURN_NONE; } +/*[clinic input] +monitoring.clear_tool_id + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_clear_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=04defc23470b1be7 input=af643d6648a66163]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + + PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (interp->monitoring_tool_names[tool_id] != NULL) { + if (_PyMonitoring_ClearToolId(tool_id) < 0) { + return NULL; + } + } + + Py_RETURN_NONE; +} + /*[clinic input] monitoring.free_tool_id @@ -2099,6 +2175,13 @@ monitoring_free_tool_id_impl(PyObject *module, int tool_id) return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (interp->monitoring_tool_names[tool_id] != NULL) { + if (_PyMonitoring_ClearToolId(tool_id) < 0) { + return NULL; + } + } + Py_CLEAR(interp->monitoring_tool_names[tool_id]); Py_RETURN_NONE; } @@ -2376,6 +2459,7 @@ monitoring__all_events_impl(PyObject *module) static PyMethodDef methods[] = { MONITORING_USE_TOOL_ID_METHODDEF + MONITORING_CLEAR_TOOL_ID_METHODDEF MONITORING_FREE_TOOL_ID_METHODDEF MONITORING_GET_TOOL_METHODDEF MONITORING_REGISTER_CALLBACK_METHODDEF diff --git a/Python/jit.c b/Python/jit.c index 33320761621c4c..234fc7dda83231 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" +#include "pycore_bitutils.h" #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_critical_section.h" @@ -113,6 +114,21 @@ mark_executable(unsigned char *memory, size_t size) // JIT compiler stuff: ///////////////////////////////////////////////////////// +#define SYMBOL_MASK_WORDS 4 + +typedef uint32_t symbol_mask[SYMBOL_MASK_WORDS]; + +typedef struct { + unsigned char *mem; + symbol_mask mask; + size_t size; +} trampoline_state; + +typedef struct { + trampoline_state trampolines; + uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; +} jit_state; + // Warning! AArch64 requires you to get your hands dirty. These are your gloves: // value[value_start : value_start + len] @@ -390,66 +406,126 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) patch_32r(location, value); } +void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); + #include "jit_stencils.h" +#if defined(__aarch64__) || defined(_M_ARM64) + #define TRAMPOLINE_SIZE 16 +#else + #define TRAMPOLINE_SIZE 0 +#endif + +// Generate and patch AArch64 trampolines. The symbols to jump to are stored +// in the jit_stencils.h in the symbols_map. +void +patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) +{ + // Masking is done modulo 32 as the mask is stored as an array of uint32_t + const uint32_t symbol_mask = 1 << (ordinal % 32); + const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; + assert(symbol_mask & trampoline_mask); + + // Count the number of set bits in the trampoline mask lower than ordinal, + // this gives the index into the array of trampolines. + int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); + for (int i = 0; i < ordinal / 32; i++) { + index += _Py_popcount32(state->trampolines.mask[i]); + } + + uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE); + assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + + uint64_t value = (uintptr_t)symbols_map[ordinal]; + + /* Generate the trampoline + 0: 58000048 ldr x8, 8 + 4: d61f0100 br x8 + 8: 00000000 // The next two words contain the 64-bit address to jump to. + c: 00000000 + */ + p[0] = 0x58000048; + p[1] = 0xD61F0100; + p[2] = value & 0xffffffff; + p[3] = value >> 32; + + patch_aarch64_26r(location, (uintptr_t)p); +} + +static void +combine_symbol_mask(const symbol_mask src, symbol_mask dest) +{ + // Calculate the union of the trampolines required by each StencilGroup + for (size_t i = 0; i < SYMBOL_MASK_WORDS; i++) { + dest[i] |= src[i]; + } +} + // Compiles executor in-place. Don't forget to call _PyJIT_Free later! int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], size_t length) { const StencilGroup *group; // Loop once to find the total compiled size: - uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; size_t code_size = 0; size_t data_size = 0; + jit_state state = {}; group = &trampoline; code_size += group->code_size; data_size += group->data_size; for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; - instruction_starts[i] = code_size; + state.instruction_starts[i] = code_size; code_size += group->code_size; data_size += group->data_size; + combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); } group = &stencil_groups[_FATAL_ERROR]; code_size += group->code_size; data_size += group->data_size; + combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); + // Calculate the size of the trampolines required by the whole trace + for (size_t i = 0; i < Py_ARRAY_LENGTH(state.trampolines.mask); i++) { + state.trampolines.size += _Py_popcount32(state.trampolines.mask[i]) * TRAMPOLINE_SIZE; + } // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); - size_t padding = page_size - ((code_size + data_size) & (page_size - 1)); - size_t total_size = code_size + data_size + padding; + size_t padding = page_size - ((code_size + data_size + state.trampolines.size) & (page_size - 1)); + size_t total_size = code_size + data_size + state.trampolines.size + padding; unsigned char *memory = jit_alloc(total_size); if (memory == NULL) { return -1; } // Update the offsets of each instruction: for (size_t i = 0; i < length; i++) { - instruction_starts[i] += (uintptr_t)memory; + state.instruction_starts[i] += (uintptr_t)memory; } // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; + state.trampolines.mem = memory + code_size + data_size; // Compile the trampoline, which handles converting between the native // calling convention and the calling convention used by jitted code // (which may be different for efficiency reasons). On platforms where // we don't change calling conventions, the trampoline is empty and // nothing is emitted here: group = &trampoline; - group->emit(code, data, executor, NULL, instruction_starts); + group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; assert(trace[0].opcode == _START_EXECUTOR); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; - group->emit(code, data, executor, instruction, instruction_starts); + group->emit(code, data, executor, instruction, &state); code += group->code_size; data += group->data_size; } // Protect against accidental buffer overrun into data: group = &stencil_groups[_FATAL_ERROR]; - group->emit(code, data, executor, NULL, instruction_starts); + group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; assert(code == memory + code_size); diff --git a/Python/lock.c b/Python/lock.c index 57675fe1873fa2..554c51d7780322 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -377,21 +377,46 @@ _PyRecursiveMutex_Lock(_PyRecursiveMutex *m) assert(m->level == 0); } +PyLockStatus +_PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (recursive_mutex_is_owned_by(m, thread)) { + m->level++; + return PY_LOCK_ACQUIRED; + } + PyLockStatus s = _PyMutex_LockTimed(&m->mutex, timeout, flags); + if (s == PY_LOCK_ACQUIRED) { + _Py_atomic_store_ullong_relaxed(&m->thread, thread); + assert(m->level == 0); + } + return s; +} + void _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m) +{ + if (_PyRecursiveMutex_TryUnlock(m) < 0) { + Py_FatalError("unlocking a recursive mutex that is not " + "owned by the current thread"); + } +} + +int +_PyRecursiveMutex_TryUnlock(_PyRecursiveMutex *m) { PyThread_ident_t thread = PyThread_get_thread_ident_ex(); if (!recursive_mutex_is_owned_by(m, thread)) { - Py_FatalError("unlocking a recursive mutex that is not owned by the" - " current thread"); + return -1; } if (m->level > 0) { m->level--; - return; + return 0; } assert(m->level == 0); _Py_atomic_store_ullong_relaxed(&m->thread, 0); PyMutex_Unlock(&m->mutex); + return 0; } #define _Py_WRITE_LOCKED 1 diff --git a/Python/marshal.c b/Python/marshal.c index b1708a7306f9e7..a280fbfd078f41 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -76,6 +76,7 @@ module marshal #define TYPE_UNKNOWN '?' #define TYPE_SET '<' #define TYPE_FROZENSET '>' +#define TYPE_SLICE ':' #define FLAG_REF '\x80' /* with a type, add obj to index */ #define TYPE_ASCII 'a' @@ -613,6 +614,13 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_pstring(view.buf, view.len, p); PyBuffer_Release(&view); } + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + W_TYPE(TYPE_SLICE, p); + w_object(slice->start, p); + w_object(slice->stop, p); + w_object(slice->step, p); + } else { W_TYPE(TYPE_UNKNOWN, p); p->error = WFERR_UNMARSHALLABLE; @@ -1218,7 +1226,7 @@ r_object(RFILE *p) v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass"); } else { - v = PyUnicode_New(0, 0); + v = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } if (v == NULL) break; @@ -1534,6 +1542,32 @@ r_object(RFILE *p) retval = Py_NewRef(v); break; + case TYPE_SLICE: + { + Py_ssize_t idx = r_ref_reserve(flag, p); + PyObject *stop = NULL; + PyObject *step = NULL; + PyObject *start = r_object(p); + if (start == NULL) { + goto cleanup; + } + stop = r_object(p); + if (stop == NULL) { + goto cleanup; + } + step = r_object(p); + if (step == NULL) { + goto cleanup; + } + retval = PySlice_New(start, stop, step); + r_ref_insert(retval, idx, flag, p); + cleanup: + Py_XDECREF(start); + Py_XDECREF(stop); + Py_XDECREF(step); + break; + } + default: /* Bogus data got written, which isn't ideal. This will let you keep working and recover. */ diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 49f01ca2932ee2..3fc9d3118d59ad 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -115,7 +115,7 @@ static void *opcode_targets[256] = { &&TARGET_UNPACK_EX, &&TARGET_UNPACK_SEQUENCE, &&TARGET_YIELD_VALUE, - &&TARGET__DO_CALL_FUNCTION_EX, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/optimizer.c b/Python/optimizer.c index 9198e410627dd4..b876b6c2bd72fd 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -533,7 +533,7 @@ translate_bytecode_to_trace( { bool first = true; PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyFunctionObject *func = _PyFrame_GetFunction(frame); assert(PyFunction_Check(func)); PyCodeObject *initial_code = code; _Py_BloomFilter_Add(dependencies, initial_code); @@ -565,6 +565,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; for (;;) { @@ -642,14 +643,12 @@ translate_bytecode_to_trace( int bitcount = _Py_popcount32(counter); int jump_likely = bitcount > 8; /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - If it's 16 or 0 (all or none were taken), adjust by 10% - (since the future is still somewhat uncertain). For values in between, adjust proportionally. */ if (jump_likely) { - confidence = confidence * (bitcount + 2) / 20; + confidence = confidence * bitcount / 16; } else { - confidence = confidence * (18 - bitcount) / 20; + confidence = confidence * (16 - bitcount) / 16; } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", @@ -1194,6 +1193,9 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil executor->jit_code = NULL; executor->jit_side_entry = NULL; executor->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + executor->vm_data.warm = true; if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); return NULL; @@ -1659,4 +1661,42 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) } } +void +_Py_Executors_InvalidateCold(PyInterpreterState *interp) +{ + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + goto error; + } + + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + + if (!exec->vm_data.warm && PyList_Append(invalidate, (PyObject *)exec) < 0) { + goto error; + } + else { + exec->vm_data.warm = false; + } + + exec = next; + } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback + _Py_Executors_InvalidateAll(interp, 0); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index f7adb44c9e09ef..25166bc2dc5c02 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -56,14 +56,14 @@ static int get_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); + return (d->_ma_watcher_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); } static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - d->ma_version_tag += (1 << DICT_MAX_WATCHERS); + d->_ma_watcher_tag += (1 << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -131,6 +131,26 @@ incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) return 0; } +static int +check_next_uop(_PyUOpInstruction *buffer, int size, int pc, uint16_t expected) +{ + if (pc + 1 >= size) { + DPRINTF(1, "Cannot rewrite %s at pc %d: buffer too small\n", + _PyOpcode_uop_name[buffer[pc].opcode], pc); + return 0; + } + uint16_t next_opcode = buffer[pc + 1].opcode; + if (next_opcode != expected) { + DPRINTF(1, + "Cannot rewrite %s at pc %d: unexpected next opcode %s, " + "expected %s\n", + _PyOpcode_uop_name[buffer[pc].opcode], pc, + _PyOpcode_uop_name[next_opcode], _PyOpcode_uop_name[expected]); + return 0; + } + return 1; +} + /* Returns 1 if successfully optimized * 0 if the trace is not suitable for optimization (yet) * -1 if there was an error. */ @@ -145,7 +165,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, return 1; } PyObject *globals = frame->f_globals; - PyFunctionObject *function = (PyFunctionObject *)frame->f_funcobj; + PyFunctionObject *function = _PyFrame_GetFunction(frame); assert(PyFunction_Check(function)); assert(function->func_builtins == builtins); assert(function->func_globals == globals); @@ -174,7 +194,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, _PyUOpInstruction *inst = &buffer[pc]; int opcode = inst->opcode; switch(opcode) { - case _GUARD_BUILTINS_VERSION: + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: if (incorrect_keys(inst, builtins)) { OPT_STAT_INC(remove_globals_incorrect_keys); return 0; @@ -182,6 +202,10 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { continue; } + if (!check_next_uop(buffer, buffer_size, pc, + _LOAD_GLOBAL_BUILTINS_FROM_KEYS)) { + continue; + } if ((builtins_watched & 1) == 0) { PyDict_Watch(BUILTINS_WATCHER_ID, builtins); builtins_watched |= 1; @@ -194,8 +218,13 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, buffer[pc].operand = function_version; function_checked |= 1; } + // We're no longer pushing the builtins keys; rewrite the + // instruction that consumed the keys to load them from the + // frame. + buffer[pc + 1].opcode = _LOAD_GLOBAL_BUILTINS; break; case _GUARD_GLOBALS_VERSION: + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: if (incorrect_keys(inst, globals)) { OPT_STAT_INC(remove_globals_incorrect_keys); return 0; @@ -204,6 +233,11 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { continue; } + if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS && + !check_next_uop(buffer, buffer_size, pc, + _LOAD_GLOBAL_MODULE_FROM_KEYS)) { + continue; + } if ((globals_watched & 1) == 0) { PyDict_Watch(GLOBALS_WATCHER_ID, globals); _Py_BloomFilter_Add(dependencies, globals); @@ -217,6 +251,12 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, buffer[pc].operand = function_version; function_checked |= 1; } + if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS) { + // We're no longer pushing the globals keys; rewrite the + // instruction that consumed the keys to load them from the + // frame. + buffer[pc + 1].opcode = _LOAD_GLOBAL_MODULE; + } break; case _LOAD_GLOBAL_BUILTINS: if (function_checked & globals_watched & builtins_watched & 1) { @@ -385,6 +425,30 @@ get_code(_PyUOpInstruction *op) return co; } +static PyCodeObject * +get_code_with_logging(_PyUOpInstruction *op) +{ + PyCodeObject *co = NULL; + uint64_t push_operand = op->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + return NULL; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + return co; +} + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9a1b9da52f4bb5..d71b55cbe1c68d 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -182,7 +182,9 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } } - res = sym_new_unknown(ctx); + else { + res = sym_new_unknown(ctx); + } } op(_BINARY_OP_ADD_INT, (left, right -- res)) { @@ -448,8 +450,10 @@ dummy_func(void) { top = bottom; } - op(_SWAP, (bottom, unused[oparg-2], top -- - top, unused[oparg-2], bottom)) { + op(_SWAP, (bottom_in, unused[oparg-2], top_in -- + top_out, unused[oparg-2], bottom_out)) { + bottom_out = bottom_in; + top_out = top_in; } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { @@ -479,9 +483,7 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { (void)owner; attr = sym_new_not_null(ctx); - if (oparg & 1) { - self_or_null = sym_new_unknown(ctx); - } + self_or_null = sym_new_unknown(ctx); } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { @@ -570,30 +572,17 @@ dummy_func(void) { op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { int argcount = oparg; - (void)callable; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - uint64_t push_operand = (this_instr + 2)->operand; - if (push_operand & 1) { - co = (PyCodeObject *)(push_operand & ~1); - DPRINTF(3, "code=%p ", co); - assert(PyCode_Check(co)); - } - else { - PyFunctionObject *func = (PyFunctionObject *)push_operand; - DPRINTF(3, "func=%p ", func); - if (func == NULL) { - DPRINTF(3, "\n"); - DPRINTF(1, "Missing function\n"); - ctx->done = true; - break; - } - co = (PyCodeObject *)func->func_code; - DPRINTF(3, "code=%p ", co); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; } + assert(self_or_null != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -619,12 +608,17 @@ dummy_func(void) { } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { - /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ - (void)callable; - (void)self_or_null; - (void)args; - new_frame = NULL; - ctx->done = true; + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = frame_new(ctx, co, 0, NULL, 0); } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -654,11 +648,10 @@ dummy_func(void) { } op(_RETURN_VALUE, (retval -- res)) { - SYNC_SP(); + SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = retval; /* Stack space handling */ assert(corresponding_check_stack == NULL); @@ -673,6 +666,8 @@ dummy_func(void) { // might be impossible, but bailing is still safe ctx->done = true; } + RELOAD_STACK(); + res = retval; } op(_RETURN_GENERATOR, ( -- res)) { @@ -841,6 +836,16 @@ dummy_func(void) { ctx->done = true; } + op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version/1 -- globals_keys)) { + globals_keys = sym_new_unknown(ctx); + (void)version; + } + + op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version/1 -- builtins_keys)) { + builtins_keys = sym_new_unknown(ctx); + (void)version; + } + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a6cfa271ae6758..6ec9e69d1dbc44 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -93,9 +93,9 @@ } case _END_SEND: { - _Py_UopsSymbol *value; - value = sym_new_not_null(ctx); - stack_pointer[-2] = value; + _Py_UopsSymbol *val; + val = sym_new_not_null(ctx); + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -630,7 +630,6 @@ ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = retval; /* Stack space handling */ assert(corresponding_check_stack == NULL); assert(co != NULL); @@ -643,6 +642,7 @@ // might be impossible, but bailing is still safe ctx->done = true; } + res = retval; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -832,9 +832,7 @@ _Py_UopsSymbol **res; _Py_UopsSymbol *null = NULL; res = &stack_pointer[0]; - for (int _i = 1; --_i >= 0;) { - res[_i] = sym_new_not_null(ctx); - } + res[0] = sym_new_not_null(ctx); null = sym_new_null(ctx); if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -846,30 +844,48 @@ break; } - case _GUARD_BUILTINS_VERSION: { + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { + _Py_UopsSymbol *globals_keys; + uint16_t version = (uint16_t)this_instr->operand; + globals_keys = sym_new_unknown(ctx); + (void)version; + stack_pointer[0] = globals_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_MODULE: { + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { + _Py_UopsSymbol *builtins_keys; + uint16_t version = (uint16_t)this_instr->operand; + builtins_keys = sym_new_unknown(ctx); + (void)version; + stack_pointer[0] = builtins_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_MODULE_FROM_KEYS: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); null = sym_new_null(ctx); - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer[-1] = res; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_BUILTINS: { + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); null = sym_new_null(ctx); - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer[-1] = res; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -1021,9 +1037,7 @@ owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); - if (oparg & 1) { - self_or_null = sym_new_unknown(ctx); - } + self_or_null = sym_new_unknown(ctx); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -1114,11 +1128,17 @@ PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; attr = sym_new_const(ctx, res); } + stack_pointer += -(oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ @@ -1239,7 +1259,11 @@ res = sym_new_type(ctx, &PyBool_Type); } else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); res = _Py_uop_sym_new_not_null(ctx); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1659,21 +1683,25 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null; _Py_UopsSymbol *callable; _Py_UOpsAbstractFrame *new_frame; - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ - (void)callable; - (void)self_or_null; - (void)args; - new_frame = NULL; - ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; - stack_pointer += -1 - oparg; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = frame_new(ctx, co, 0, NULL, 0); + stack_pointer[0] = (_Py_UopsSymbol *)new_frame; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1687,14 +1715,12 @@ } case _EXPAND_METHOD: { - _Py_UopsSymbol *method; + _Py_UopsSymbol **method; _Py_UopsSymbol **self; + method = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; - method = sym_new_not_null(ctx); - for (int _i = 1; --_i >= 0;) { - self[_i] = sym_new_not_null(ctx); - } - stack_pointer[-2 - oparg] = method; + method[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); break; } @@ -1771,23 +1797,12 @@ (void)callable; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - uint64_t push_operand = (this_instr + 2)->operand; - if (push_operand & 1) { - co = (PyCodeObject *)(push_operand & ~1); - DPRINTF(3, "code=%p ", co); - assert(PyCode_Check(co)); - } - else { - PyFunctionObject *func = (PyFunctionObject *)push_operand; - DPRINTF(3, "func=%p ", func); - if (func == NULL) { - DPRINTF(3, "\n"); - DPRINTF(1, "Missing function\n"); - ctx->done = true; - break; - } - co = (PyCodeObject *)func->func_code; - DPRINTF(3, "code=%p ", co); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; } assert(self_or_null != NULL); assert(args != NULL); @@ -1801,8 +1816,8 @@ } else { new_frame = frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; - stack_pointer += -1 - oparg; + stack_pointer[0] = (_Py_UopsSymbol *)new_frame; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1835,9 +1850,11 @@ if (first_valid_check_stack == NULL) { first_valid_check_stack = corresponding_check_stack; } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; + else { + if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } } corresponding_check_stack = NULL; break; @@ -2015,6 +2032,24 @@ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD_KW: { + _Py_UopsSymbol **func; + _Py_UopsSymbol **maybe_self; + _Py_UopsSymbol **args; + _Py_UopsSymbol *kwnames_out; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - oparg]; + func[0] = sym_new_not_null(ctx); + maybe_self[0] = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + kwnames_out = sym_new_not_null(ctx); + stack_pointer[-1] = kwnames_out; + break; + } + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { @@ -2048,17 +2083,12 @@ } case _EXPAND_METHOD_KW: { - _Py_UopsSymbol *method; + _Py_UopsSymbol **method; _Py_UopsSymbol **self; - _Py_UopsSymbol *kwnames; + method = &stack_pointer[-3 - oparg]; self = &stack_pointer[-2 - oparg]; - method = sym_new_not_null(ctx); - for (int _i = 1; --_i >= 0;) { - self[_i] = sym_new_not_null(ctx); - } - kwnames = sym_new_not_null(ctx); - stack_pointer[-3 - oparg] = method; - stack_pointer[-1] = kwnames; + method[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); break; } @@ -2077,7 +2107,17 @@ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ - /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + case _MAKE_CALLARGS_A_TUPLE: { + _Py_UopsSymbol *tuple; + _Py_UopsSymbol *kwargs_out = NULL; + tuple = sym_new_not_null(ctx); + kwargs_out = sym_new_not_null(ctx); + stack_pointer[-1 - (oparg & 1)] = tuple; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; + break; + } + + /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; @@ -2087,9 +2127,9 @@ } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsSymbol *func_st; - func_st = sym_new_not_null(ctx); - stack_pointer[-2] = func_st; + _Py_UopsSymbol *func_out; + func_out = sym_new_not_null(ctx); + stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2108,14 +2148,14 @@ assert(framesize > 0); assert(framesize <= curr_space); curr_space -= framesize; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe ctx->done = true; } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2184,7 +2224,9 @@ res = sym_new_type(ctx, &PyFloat_Type); } } - res = sym_new_unknown(ctx); + else { + res = sym_new_unknown(ctx); + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2192,12 +2234,16 @@ } case _SWAP: { - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + _Py_UopsSymbol *top_in; + _Py_UopsSymbol *bottom_in; + _Py_UopsSymbol *top_out; + _Py_UopsSymbol *bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; break; } @@ -2223,7 +2269,11 @@ if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, value != Py_True); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2236,7 +2286,11 @@ if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, value != Py_False); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2249,14 +2303,22 @@ if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + else { + if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, true); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2266,14 +2328,22 @@ if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + else { + if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, false); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2367,6 +2437,30 @@ break; } + case _LOAD_GLOBAL_MODULE: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _INTERNAL_INCREMENT_OPT_COUNTER: { stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2381,6 +2475,10 @@ break; } + case _MAKE_WARM: { + break; + } + case _FATAL_ERROR: { break; } diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 841b1d71ea16cb..a7e9760e35d87a 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -102,7 +102,14 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) millis = INFINITE; } else { - millis = (DWORD) (timeout / 1000000); + PyTime_t div = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_TIMEOUT); + // Prevent overflow with clamping the result + if ((PyTime_t)PY_DWORD_MAX < div) { + millis = PY_DWORD_MAX; + } + else { + millis = (DWORD) div; + } } wait = WaitForSingleObjectEx(sema->platform_sem, millis, FALSE); if (wait == WAIT_OBJECT_0) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 27faf723745c21..ebeee4f41d795d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -28,7 +28,7 @@ #include "pycore_sliceobject.h" // _PySlice_Fini() #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() -#include "pycore_typeid.h" // _PyType_FinalizeIdPool() +#include "pycore_uniqueid.h" // _PyType_FinalizeIdPool() #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() @@ -1834,7 +1834,7 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_Fini(interp); #ifdef Py_GIL_DISABLED - _PyType_FinalizeIdPool(interp); + _PyObject_FinalizeUniqueIdPool(interp); #endif _PyCode_Fini(interp); @@ -2020,7 +2020,7 @@ _Py_Finalize(_PyRuntimeState *runtime) /* Ensure that remaining threads are detached */ _PyEval_StopTheWorldAll(runtime); - /* Remaining daemon threads will automatically exit + /* Remaining daemon threads will be trapped in PyThread_hang_thread when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ _PyInterpreterState_SetFinalizing(tstate->interp, tstate); _PyRuntimeState_SetFinalizing(runtime, tstate); @@ -2503,18 +2503,12 @@ finalize_subinterpreters(void) static PyStatus add_main_module(PyInterpreterState *interp) { - PyObject *m, *d, *ann_dict; + PyObject *m, *d; m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); d = PyModule_GetDict(m); - ann_dict = PyDict_New(); - if ((ann_dict == NULL) || - (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) { - return _PyStatus_ERR("Failed to initialize __main__.__annotations__"); - } - Py_DECREF(ann_dict); int has_builtins = PyDict_ContainsString(d, "__builtins__"); if (has_builtins < 0) { diff --git a/Python/pystate.c b/Python/pystate.c index 6bf7ebeb75ff73..5d94b7714bd607 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -20,7 +20,7 @@ #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() -#include "pycore_typeid.h" // _PyType_FinalizeThreadLocalRefcounts() +#include "pycore_uniqueid.h" // _PyType_FinalizeThreadLocalRefcounts() /* -------------------------------------------------------------------------- CAUTION @@ -523,12 +523,6 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) _PyTypes_AfterFork(); - /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does - * not force the default allocator. */ - if (_PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex) < 0) { - return _PyStatus_ERR("Failed to reinitialize runtime locks"); - } - PyStatus status = gilstate_tss_reinit(runtime); if (_PyStatus_EXCEPTION(status)) { return status; @@ -629,6 +623,8 @@ init_interpreter(PyInterpreterState *interp, assert(id > 0 || (id == 0 && interp == runtime->interpreters.main)); interp->id = id; + interp->id_refcount = 0; + assert(runtime->interpreters.head == interp); assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; @@ -654,12 +650,14 @@ init_interpreter(PyInterpreterState *interp, interp->monitoring_callables[t][e] = NULL; } + interp->monitoring_tool_versions[t] = 0; } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; #ifdef _Py_TIER2 (void)_Py_SetOptimizer(interp, NULL); interp->executor_list_head = NULL; + interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; #endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ @@ -987,10 +985,6 @@ PyInterpreterState_Delete(PyInterpreterState *interp) } HEAD_UNLOCK(runtime); - if (interp->id_mutex != NULL) { - PyThread_free_lock(interp->id_mutex); - } - _Py_qsbr_fini(interp); _PyObject_FiniState(interp); @@ -1029,9 +1023,6 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) // the "current" tstate to be set? PyInterpreterState_Clear(interp); // XXX must activate? zapthreads(interp); - if (interp->id_mutex != NULL) { - PyThread_free_lock(interp->id_mutex); - } PyInterpreterState *prev_interp = interp; interp = interp->next; free_interpreter(prev_interp); @@ -1245,9 +1236,6 @@ PyInterpreterState_GetID(PyInterpreterState *interp) PyObject * _PyInterpreterState_GetIDObject(PyInterpreterState *interp) { - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; int64_t interpid = interp->id; if (interpid < 0) { return NULL; @@ -1257,50 +1245,22 @@ _PyInterpreterState_GetIDObject(PyInterpreterState *interp) } -int -_PyInterpreterState_IDInitref(PyInterpreterState *interp) -{ - if (interp->id_mutex != NULL) { - return 0; - } - interp->id_mutex = PyThread_allocate_lock(); - if (interp->id_mutex == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "failed to create init interpreter ID mutex"); - return -1; - } - interp->id_refcount = 0; - return 0; -} - -int +void _PyInterpreterState_IDIncref(PyInterpreterState *interp) { - if (_PyInterpreterState_IDInitref(interp) < 0) { - return -1; - } - - PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); - interp->id_refcount += 1; - PyThread_release_lock(interp->id_mutex); - return 0; + _Py_atomic_add_ssize(&interp->id_refcount, 1); } void _PyInterpreterState_IDDecref(PyInterpreterState *interp) { - assert(interp->id_mutex != NULL); _PyRuntimeState *runtime = interp->runtime; - PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); - assert(interp->id_refcount != 0); - interp->id_refcount -= 1; - int64_t refcount = interp->id_refcount; - PyThread_release_lock(interp->id_mutex); + Py_ssize_t refcount = _Py_atomic_add_ssize(&interp->id_refcount, -1); - if (refcount == 0 && interp->requires_idref) { + if (refcount == 1 && interp->requires_idref) { PyThreadState *tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); @@ -1744,7 +1704,7 @@ PyThreadState_Clear(PyThreadState *tstate) // Merge our thread-local refcounts into the type's own refcount and // free our local refcount array. - _PyType_FinalizeThreadLocalRefcounts((_PyThreadStateImpl *)tstate); + _PyObject_FinalizePerThreadRefcounts((_PyThreadStateImpl *)tstate); // Remove ourself from the biased reference counting table of threads. _Py_brc_remove_thread(tstate); @@ -1804,7 +1764,7 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; tstate->interp->object_state.reftotal += tstate_impl->reftotal; tstate_impl->reftotal = 0; - assert(tstate_impl->types.refcounts == NULL); + assert(tstate_impl->refcounts.values == NULL); #endif HEAD_UNLOCK(runtime); diff --git a/Python/specialize.c b/Python/specialize.c index d8bff39511cf12..4b33a468733d6b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1599,7 +1599,7 @@ function_get_version(PyObject *o, int opcode) assert(Py_IS_TYPE(o, &PyFunction_Type)); PyFunctionObject *func = (PyFunctionObject *)o; uint32_t version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { + if (!_PyFunction_IsVersionValid(version)) { SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } @@ -1692,7 +1692,7 @@ _Py_Specialize_BinarySubscr( goto fail; } uint32_t version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { + if (!_PyFunction_IsVersionValid(version)) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } @@ -1977,7 +1977,7 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, argcount = code->co_argcount; } int version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { + if (!_PyFunction_IsVersionValid(version)) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } @@ -2009,7 +2009,7 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } int version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { + if (!_PyFunction_IsVersionValid(version)) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } diff --git a/Python/symtable.c b/Python/symtable.c index 8bc9db6d7d6811..709918b27afcc8 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -354,7 +354,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) static void dump_symtable(PySTEntryObject* ste) { - PyObject *empty = PyUnicode_FromString(""); + PyObject *empty = Py_GetConstant(Py_CONSTANT_EMPTY_STR); assert(empty != NULL); _dump_symtable(ste, empty); Py_DECREF(empty); @@ -3120,33 +3120,30 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) if (ipriv == plen) { return Py_NewRef(ident); /* Don't mangle if class is just underscores */ } - plen -= ipriv; - if (plen + nlen >= PY_SSIZE_T_MAX - 1) { + if (nlen + (plen - ipriv) >= PY_SSIZE_T_MAX - 1) { PyErr_SetString(PyExc_OverflowError, "private identifier too large to be mangled"); return NULL; } - Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(ident); - if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar) { - maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj); - } - - PyObject *result = PyUnicode_New(1 + nlen + plen, maxchar); - if (!result) { + PyUnicodeWriter *writer = PyUnicodeWriter_Create(1 + nlen + (plen - ipriv)); + if (!writer) { return NULL; } - /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ - PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_'); - if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) { - Py_DECREF(result); - return NULL; + // ident = "_" + priv[ipriv:] + ident + if (PyUnicodeWriter_WriteChar(writer, '_') < 0) { + goto error; } - if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) { - Py_DECREF(result); - return NULL; + if (PyUnicodeWriter_WriteSubstring(writer, privateobj, ipriv, plen) < 0) { + goto error; } - assert(_PyUnicode_CheckConsistency(result, 1)); - return result; + if (PyUnicodeWriter_WriteStr(writer, ident) < 0) { + goto error; + } + return PyUnicodeWriter_Finish(writer); + +error: + PyUnicodeWriter_Discard(writer); + return NULL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 887a916563a2e1..ac343a8048e008 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2384,10 +2384,11 @@ sys__getframemodulename_impl(PyObject *module, int depth) while (f && (_PyFrame_IsIncomplete(f) || depth-- > 0)) { f = f->previous; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + PyObject *r = PyFunction_GetModule(func); if (!r) { PyErr_Clear(); r = Py_None; diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 425658131c2fce..3a01a7fe327c8f 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -291,6 +291,14 @@ PyThread_exit_thread(void) _endthreadex(0); } +void _Py_NO_RETURN +PyThread_hang_thread(void) +{ + while (1) { + SleepEx(INFINITE, TRUE); + } +} + /* * Lock support. It has to be implemented as semaphores. * I [Dag] tried to implement it with mutex but I could find a way to diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index f588b4620da0d3..c010b3a827757f 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -16,6 +16,7 @@ #undef destructor #endif #include +#include /* pause(), also getthrid() on OpenBSD */ #if defined(__linux__) # include /* syscall(SYS_gettid) */ @@ -23,8 +24,6 @@ # include /* pthread_getthreadid_np() */ #elif defined(__FreeBSD_kernel__) # include /* syscall(SYS_thr_self) */ -#elif defined(__OpenBSD__) -# include /* getthrid() */ #elif defined(_AIX) # include /* thread_self() */ #elif defined(__NetBSD__) @@ -419,6 +418,18 @@ PyThread_exit_thread(void) #endif } +void _Py_NO_RETURN +PyThread_hang_thread(void) +{ + while (1) { +#if defined(__wasi__) + sleep(9999999); // WASI doesn't have pause() ?! +#else + pause(); +#endif + } +} + #ifdef USE_SEMAPHORES /* diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h index 4741e594e52e65..458f8fc5951720 100644 --- a/Python/thread_pthread_stubs.h +++ b/Python/thread_pthread_stubs.h @@ -1,5 +1,9 @@ #include "cpython/pthread_stubs.h" +typedef struct py_stub_tls_entry py_tls_entry; + +#define py_tls_entries (_PyRuntime.threads.stubs.tls_entries) + // mutex int pthread_mutex_init(pthread_mutex_t *restrict mutex, @@ -105,7 +109,7 @@ pthread_join(pthread_t thread, void** value_ptr) PyAPI_FUNC(pthread_t) pthread_self(void) { - return 0; + return (pthread_t)(uintptr_t)&py_tls_entries; } int @@ -134,10 +138,6 @@ pthread_attr_destroy(pthread_attr_t *attr) } -typedef struct py_stub_tls_entry py_tls_entry; - -#define py_tls_entries (_PyRuntime.threads.stubs.tls_entries) - int pthread_key_create(pthread_key_t *key, void (*destr_function)(void *)) { diff --git a/Python/typeid.c b/Python/uniqueid.c similarity index 51% rename from Python/typeid.c rename to Python/uniqueid.c index 83a68723ded61b..9a9ee2f39467b0 100644 --- a/Python/typeid.c +++ b/Python/uniqueid.c @@ -3,12 +3,14 @@ #include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_object.h" // _Py_IncRefTotal -#include "pycore_typeid.h" +#include "pycore_uniqueid.h" -// This contains code for allocating unique ids to heap type objects -// and re-using those ids when the type is deallocated. +// This contains code for allocating unique ids for per-thread reference +// counting and re-using those ids when an object is deallocated. // -// See Include/internal/pycore_typeid.h for more details. +// Currently, per-thread reference counting is only used for heap types. +// +// See Include/internal/pycore_uniqueid.h for more details. #ifdef Py_GIL_DISABLED @@ -18,7 +20,7 @@ #define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) static int -resize_interp_type_id_pool(struct _Py_type_id_pool *pool) +resize_interp_type_id_pool(struct _Py_unique_id_pool *pool) { if ((size_t)pool->size > PY_SSIZE_T_MAX / (2 * sizeof(*pool->table))) { return -1; @@ -29,8 +31,8 @@ resize_interp_type_id_pool(struct _Py_type_id_pool *pool) new_size = POOL_MIN_SIZE; } - _Py_type_id_entry *table = PyMem_Realloc(pool->table, - new_size * sizeof(*pool->table)); + _Py_unique_id_entry *table = PyMem_Realloc(pool->table, + new_size * sizeof(*pool->table)); if (table == NULL) { return -1; } @@ -50,70 +52,67 @@ resize_interp_type_id_pool(struct _Py_type_id_pool *pool) static int resize_local_refcounts(_PyThreadStateImpl *tstate) { - if (tstate->types.is_finalized) { + if (tstate->refcounts.is_finalized) { return -1; } - struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; + struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids; Py_ssize_t size = _Py_atomic_load_ssize(&pool->size); - Py_ssize_t *refcnts = PyMem_Realloc(tstate->types.refcounts, + Py_ssize_t *refcnts = PyMem_Realloc(tstate->refcounts.values, size * sizeof(Py_ssize_t)); if (refcnts == NULL) { return -1; } - Py_ssize_t old_size = tstate->types.size; + Py_ssize_t old_size = tstate->refcounts.size; if (old_size < size) { memset(refcnts + old_size, 0, (size - old_size) * sizeof(Py_ssize_t)); } - tstate->types.refcounts = refcnts; - tstate->types.size = size; + tstate->refcounts.values = refcnts; + tstate->refcounts.size = size; return 0; } -void -_PyType_AssignId(PyHeapTypeObject *type) +Py_ssize_t +_PyObject_AssignUniqueId(PyObject *obj) { PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_type_id_pool *pool = &interp->type_ids; + struct _Py_unique_id_pool *pool = &interp->unique_ids; LOCK_POOL(pool); if (pool->freelist == NULL) { if (resize_interp_type_id_pool(pool) < 0) { - type->unique_id = -1; UNLOCK_POOL(pool); - return; + return -1; } } - _Py_type_id_entry *entry = pool->freelist; + _Py_unique_id_entry *entry = pool->freelist; pool->freelist = entry->next; - entry->type = type; - _PyObject_SetDeferredRefcount((PyObject *)type); - type->unique_id = (entry - pool->table); + entry->obj = obj; + _PyObject_SetDeferredRefcount(obj); + Py_ssize_t unique_id = (entry - pool->table); UNLOCK_POOL(pool); + return unique_id; } void -_PyType_ReleaseId(PyHeapTypeObject *type) +_PyObject_ReleaseUniqueId(Py_ssize_t unique_id) { PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_type_id_pool *pool = &interp->type_ids; + struct _Py_unique_id_pool *pool = &interp->unique_ids; - if (type->unique_id < 0) { - // The type doesn't have an id assigned. + if (unique_id < 0) { + // The id is not assigned return; } LOCK_POOL(pool); - _Py_type_id_entry *entry = &pool->table[type->unique_id]; - assert(entry->type == type); + _Py_unique_id_entry *entry = &pool->table[unique_id]; entry->next = pool->freelist; pool->freelist = entry; - - type->unique_id = -1; UNLOCK_POOL(pool); } @@ -127,8 +126,8 @@ _PyType_IncrefSlow(PyHeapTypeObject *type) return; } - assert(type->unique_id < tstate->types.size); - tstate->types.refcounts[type->unique_id]++; + assert(type->unique_id < tstate->refcounts.size); + tstate->refcounts.values[type->unique_id]++; #ifdef Py_REF_DEBUG _Py_IncRefTotal((PyThreadState *)tstate); #endif @@ -136,59 +135,64 @@ _PyType_IncrefSlow(PyHeapTypeObject *type) } void -_PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate) +_PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate) { - if (tstate->types.refcounts == NULL) { + if (tstate->refcounts.values == NULL) { return; } - struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; + struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids; LOCK_POOL(pool); - for (Py_ssize_t i = 0, n = tstate->types.size; i < n; i++) { - Py_ssize_t refcnt = tstate->types.refcounts[i]; + for (Py_ssize_t i = 0, n = tstate->refcounts.size; i < n; i++) { + Py_ssize_t refcnt = tstate->refcounts.values[i]; if (refcnt != 0) { - PyObject *type = (PyObject *)pool->table[i].type; - assert(PyType_Check(type)); - - _Py_atomic_add_ssize(&type->ob_ref_shared, + PyObject *obj = pool->table[i].obj; + _Py_atomic_add_ssize(&obj->ob_ref_shared, refcnt << _Py_REF_SHARED_SHIFT); - tstate->types.refcounts[i] = 0; + tstate->refcounts.values[i] = 0; } } UNLOCK_POOL(pool); } void -_PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate) +_PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate) { - _PyType_MergeThreadLocalRefcounts(tstate); + _PyObject_MergePerThreadRefcounts(tstate); - PyMem_Free(tstate->types.refcounts); - tstate->types.refcounts = NULL; - tstate->types.size = 0; - tstate->types.is_finalized = 1; + PyMem_Free(tstate->refcounts.values); + tstate->refcounts.values = NULL; + tstate->refcounts.size = 0; + tstate->refcounts.is_finalized = 1; } void -_PyType_FinalizeIdPool(PyInterpreterState *interp) +_PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp) { - struct _Py_type_id_pool *pool = &interp->type_ids; + struct _Py_unique_id_pool *pool = &interp->unique_ids; // First, set the free-list to NULL values while (pool->freelist) { - _Py_type_id_entry *next = pool->freelist->next; - pool->freelist->type = NULL; + _Py_unique_id_entry *next = pool->freelist->next; + pool->freelist->obj = NULL; pool->freelist = next; } // Now everything non-NULL is a type. Set the type's id to -1 in case it // outlives the interpreter. for (Py_ssize_t i = 0; i < pool->size; i++) { - PyHeapTypeObject *ht = pool->table[i].type; - if (ht) { - ht->unique_id = -1; - pool->table[i].type = NULL; + PyObject *obj = pool->table[i].obj; + pool->table[i].obj = NULL; + if (obj == NULL) { + continue; + } + if (PyType_Check(obj)) { + assert(PyType_HasFeature((PyTypeObject *)obj, Py_TPFLAGS_HEAPTYPE)); + ((PyHeapTypeObject *)obj)->unique_id = -1; + } + else { + Py_UNREACHABLE(); } } PyMem_Free(pool->table); diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 882918fafb1edd..b5b6de0e7dc2dc 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -433,7 +433,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': # Give a nice message for common mistakes. # To cover tricky cases (like "\n") we also generate C asserts. raise ValueError( - 'do not use &_PyID or &_Py_STR for one-character latin-1 ' + 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name @@ -442,7 +442,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': overlap = identifiers & set(strings.keys()) if overlap: raise ValueError( - 'do not use both _PyID and _Py_DECLARE_STR for the same string: ' + 'do not use both _Py_ID and _Py_DECLARE_STR for the same string: ' + repr(overlap)) return identifiers, strings diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index e34a36c1a573e5..1a24b07c3ff707 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -5,12 +5,10 @@ set -e -x # The check_generated_files job of .github/workflows/build.yml must kept in # sync with this script. Use the same container image than the job so the job # doesn't need to run autoreconf in a container. -IMAGE="ubuntu:22.04" -DEPENDENCIES="autotools-dev autoconf autoconf-archive pkg-config" +IMAGE="ghcr.io/python/autoconf:2024.10.11.11293396815" AUTORECONF="autoreconf -ivf -Werror" WORK_DIR="/src" -SHELL_CMD="apt-get update && apt-get -yq install $DEPENDENCIES && cd $WORK_DIR && $AUTORECONF" abs_srcdir=$(cd $(dirname $0)/../..; pwd) @@ -28,4 +26,4 @@ if command -v selinuxenabled >/dev/null && selinuxenabled; then PATH_OPT=":Z" fi -"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" /usr/bin/bash -c "$SHELL_CMD" +"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e1c07f88b963bc..badd7b79102310 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -358,7 +358,6 @@ Modules/_testclinic.c - TestClass - ##----------------------- ## static types -Modules/_cursesmodule.c - PyCursesWindow_Type - Modules/_datetimemodule.c - PyDateTime_DateTimeType - Modules/_datetimemodule.c - PyDateTime_DateType - Modules/_datetimemodule.c - PyDateTime_DeltaType - @@ -383,7 +382,6 @@ Modules/_tkinter.c - Tktt_Type - Modules/xxlimited_35.c - Xxo_Type - ## exception types -Modules/_cursesmodule.c - PyCursesError - Modules/_tkinter.c - Tkinter_TclError - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - @@ -409,6 +407,7 @@ Modules/_tkinter.c - trbInCmd - Include/datetime.h - PyDateTimeAPI - Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized - Modules/_ctypes/malloc_closure.c - _pagesize - +Modules/_cursesmodule.c - curses_module_loaded - Modules/_cursesmodule.c - curses_initscr_called - Modules/_cursesmodule.c - curses_setupterm_called - Modules/_cursesmodule.c - curses_start_color_called - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index f4dc807198a8ef..e6c599a2ac4a46 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -345,6 +345,7 @@ Python/ast_opt.c fold_unaryop ops - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - +Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LOCATION - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3cc36b6b5841bd..381ad3a4e2082c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,13 +1,13 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +import itertools import lexer import parser import re from typing import Optional - @dataclass class Properties: - escapes: bool + escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] error_with_pop: bool error_without_pop: bool deopts: bool @@ -29,14 +29,21 @@ class Properties: needs_prev: bool = False def dump(self, indent: str) -> None: - print(indent, end="") - text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) + simple_properties = self.__dict__.copy() + del simple_properties["escaping_calls"] + text = "escaping_calls:\n" + for tkns in self.escaping_calls.values(): + text += f"{indent} {tkns}\n" + text += ", ".join([f"{key}: {value}" for (key, value) in simple_properties.items()]) print(indent, text, sep="") @staticmethod def from_list(properties: list["Properties"]) -> "Properties": + escaping_calls: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} + for p in properties: + escaping_calls.update(p.escaping_calls) return Properties( - escapes=any(p.escapes for p in properties), + escaping_calls=escaping_calls, error_with_pop=any(p.error_with_pop for p in properties), error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), @@ -59,9 +66,12 @@ def from_list(properties: list["Properties"]) -> "Properties": def infallible(self) -> bool: return not self.error_with_pop and not self.error_without_pop + @property + def escapes(self) -> bool: + return bool(self.escaping_calls) SKIP_PROPERTIES = Properties( - escapes=False, + escaping_calls={}, error_with_pop=False, error_without_pop=False, deopts=False, @@ -156,6 +166,7 @@ class Uop: stack: StackEffect caches: list[CacheEntry] deferred_refs: dict[lexer.Token, str | None] + output_stores: list[lexer.Token] body: list[lexer.Token] properties: Properties _size: int = -1 @@ -248,6 +259,7 @@ class PseudoInstruction: name: str stack: StackEffect targets: list[Instruction] + as_sequence: bool flags: list[str] opcode: int = -1 @@ -321,11 +333,24 @@ def analyze_stack( ] # Mark variables with matching names at the base of the stack as "peek" modified = False - for input, output in zip(inputs, outputs): - if input.name == output.name and not modified: - input.peek = output.peek = True + input_names: dict[str, lexer.Token] = { i.name : i.first_token for i in op.inputs if i.name != "unused" } + for input, output in itertools.zip_longest(inputs, outputs): + if output is None: + pass + elif input is None: + if output.name in input_names: + raise analysis_error( + f"Reuse of variable '{output.name}' at different stack location", + input_names[output.name]) + elif input.name == output.name: + if not modified: + input.peek = output.peek = True else: modified = True + if output.name in input_names: + raise analysis_error( + f"Reuse of variable '{output.name}' at different stack location", + input_names[output.name]) if isinstance(op, parser.InstDef): output_names = [out.name for out in outputs] for input in inputs: @@ -353,17 +378,50 @@ def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: return [CacheEntry(i.name, int(i.size)) for i in caches] +def find_assignment_target(node: parser.InstDef, idx: int) -> list[lexer.Token]: + """Find the tokens that make up the left-hand side of an assignment""" + offset = 0 + for tkn in reversed(node.block.tokens[: idx]): + if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + return node.block.tokens[idx - offset : idx] + offset += 1 + return [] + + +def find_stores_outputs(node: parser.InstDef) -> list[lexer.Token]: + res: list[lexer.Token] = [] + outnames = { out.name for out in node.outputs } + innames = { out.name for out in node.inputs } + for idx, tkn in enumerate(node.block.tokens): + if tkn.kind == "AND": + name = node.block.tokens[idx+1] + if name.text in outnames: + res.append(name) + if tkn.kind != "EQUALS": + continue + lhs = find_assignment_target(node, idx) + assert lhs + while lhs and lhs[0].kind == "COMMENT": + lhs = lhs[1:] + if len(lhs) != 1 or lhs[0].kind != "IDENTIFIER": + continue + name = lhs[0] + if name.text in innames: + raise analysis_error(f"Cannot assign to input variable '{name.text}'", name) + if name.text in outnames: + res.append(name) + return res + def analyze_deferred_refs(node: parser.InstDef) -> dict[lexer.Token, str | None]: """Look for PyStackRef_FromPyObjectNew() calls""" - def find_assignment_target(idx: int) -> list[lexer.Token]: - """Find the tokens that make up the left-hand side of an assignment""" - offset = 1 + def in_frame_push(idx: int) -> bool: for tkn in reversed(node.block.tokens[: idx - 1]): - if tkn.kind == "SEMI" or tkn.kind == "LBRACE" or tkn.kind == "RBRACE": - return node.block.tokens[idx - offset : idx - 1] - offset += 1 - return [] + if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + return False + if tkn.kind == "IDENTIFIER" and tkn.text == "_PyFrame_PushUnchecked": + return True + return False refs: dict[lexer.Token, str | None] = {} for idx, tkn in enumerate(node.block.tokens): @@ -371,9 +429,13 @@ def find_assignment_target(idx: int) -> list[lexer.Token]: continue if idx == 0 or node.block.tokens[idx - 1].kind != "EQUALS": + if in_frame_push(idx): + # PyStackRef_FromPyObjectNew() is called in _PyFrame_PushUnchecked() + refs[tkn] = None + continue raise analysis_error("Expected '=' before PyStackRef_FromPyObjectNew", tkn) - lhs = find_assignment_target(idx) + lhs = find_assignment_target(node, idx - 1) if len(lhs) == 0: raise analysis_error( "PyStackRef_FromPyObjectNew() must be assigned to an output", tkn @@ -393,9 +455,13 @@ def find_assignment_target(idx: int) -> list[lexer.Token]: ) name = lhs[0].text - if not any(var.name == name for var in node.outputs): + match = ( + any(var.name == name for var in node.inputs) + or any(var.name == name for var in node.outputs) + ) + if not match: raise analysis_error( - f"PyStackRef_FromPyObjectNew() must be assigned to an output, not '{name}'", + f"PyStackRef_FromPyObjectNew() must be assigned to an input or output, not '{name}'", tkn, ) @@ -448,124 +514,204 @@ def has_error_without_pop(op: parser.InstDef) -> bool: NON_ESCAPING_FUNCTIONS = ( - "PyStackRef_FromPyObjectSteal", + "PyCFunction_GET_FLAGS", + "PyCFunction_GET_FUNCTION", + "PyCFunction_GET_SELF", + "PyCell_GetRef", + "PyCell_New", + "PyCell_SwapTakeRef", + "PyExceptionInstance_Class", + "PyException_GetCause", + "PyException_GetContext", + "PyException_GetTraceback", + "PyFloat_AS_DOUBLE", + "PyFloat_FromDouble", + "PyFunction_GET_CODE", + "PyFunction_GET_GLOBALS", + "PyList_GET_ITEM", + "PyList_GET_SIZE", + "PyList_SET_ITEM", + "PyLong_AsLong", + "PyLong_FromLong", + "PyLong_FromSsize_t", + "PySlice_New", "PyStackRef_AsPyObjectBorrow", + "PyStackRef_AsPyObjectNew", "PyStackRef_AsPyObjectSteal", + "PyStackRef_CLEAR", "PyStackRef_CLOSE", + "PyStackRef_CLOSE_SPECIALIZED", "PyStackRef_DUP", - "PyStackRef_CLEAR", + "PyStackRef_False", + "PyStackRef_FromPyObjectImmortal", + "PyStackRef_FromPyObjectNew", + "PyStackRef_FromPyObjectSteal", + "PyStackRef_Is", "PyStackRef_IsNull", + "PyStackRef_None", "PyStackRef_TYPE", - "PyStackRef_False", "PyStackRef_True", - "PyStackRef_None", - "PyStackRef_Is", - "PyStackRef_FromPyObjectNew", - "PyStackRef_AsPyObjectNew", - "PyStackRef_FromPyObjectImmortal", - "Py_INCREF", - "_PyManagedDictPointer_IsValues", - "_PyObject_GetManagedDict", - "_PyObject_ManagedDictPointer", - "_PyObject_InlineValues", - "_PyDictValues_AddToInsertionOrder", - "Py_DECREF", - "Py_XDECREF", - "_Py_DECREF_SPECIALIZED", - "DECREF_INPUTS_AND_REUSE_FLOAT", - "PyUnicode_Append", - "_PyLong_IsZero", - "Py_SIZE", - "Py_TYPE", - "PyList_GET_ITEM", - "PyList_SET_ITEM", "PyTuple_GET_ITEM", - "PyList_GET_SIZE", "PyTuple_GET_SIZE", + "PyType_HasFeature", + "PyUnicode_Append", + "PyUnicode_Concat", + "PyUnicode_GET_LENGTH", + "PyUnicode_READ_CHAR", "Py_ARRAY_LENGTH", + "Py_CLEAR", + "Py_DECREF", + "Py_FatalError", + "Py_INCREF", + "Py_IS_TYPE", + "Py_NewRef", + "Py_REFCNT", + "Py_SIZE", + "Py_TYPE", + "Py_UNREACHABLE", "Py_Unicode_GET_LENGTH", - "PyUnicode_READ_CHAR", - "_Py_SINGLETON", - "PyUnicode_GET_LENGTH", - "_PyLong_IsCompact", - "_PyLong_IsNonNegativeCompact", + "Py_XDECREF", + "_PyCode_CODE", + "_PyDictValues_AddToInsertionOrder", + "_PyErr_Occurred", + "_PyEval_FrameClearAndPop", + "_PyFloat_FromDouble_ConsumeInputs", + "_PyFrame_GetCode", + "_PyFrame_IsIncomplete", + "_PyFrame_PushUnchecked", + "_PyFrame_SetStackPointer", + "_PyFrame_StackPush", + "_PyFunction_SetVersion", + "_PyGen_GetGeneratorFromFrame", + "_PyInterpreterState_GET", + "_PyList_AppendTakeRef", + "_PyList_FromStackRefSteal", + "_PyList_ITEMS", + "_PyLong_Add", "_PyLong_CompactValue", "_PyLong_DigitCount", - "_Py_NewRef", - "_Py_IsImmortal", - "PyLong_FromLong", - "_Py_STR", - "_PyLong_Add", + "_PyLong_IsCompact", + "_PyLong_IsNonNegativeCompact", + "_PyLong_IsZero", "_PyLong_Multiply", "_PyLong_Subtract", - "Py_NewRef", - "_PyList_ITEMS", - "_PyTuple_ITEMS", - "_PyList_AppendTakeRef", - "_Py_atomic_load_uintptr_relaxed", - "_PyFrame_GetCode", + "_PyManagedDictPointer_IsValues", + "_PyObject_GC_IS_TRACKED", + "_PyObject_GC_MAY_BE_TRACKED", + "_PyObject_GC_TRACK", + "_PyObject_GetManagedDict", + "_PyObject_InlineValues", + "_PyObject_ManagedDictPointer", "_PyThreadState_HasStackSpace", - "_PyUnicode_Equal", - "_PyFrame_SetStackPointer", + "_PyTuple_FromArraySteal", + "_PyTuple_FromStackRefSteal", + "_PyTuple_ITEMS", "_PyType_HasFeature", - "PyUnicode_Concat", - "PySlice_New", + "_PyType_NewManagedObject", + "_PyUnicode_Equal", + "_PyUnicode_JoinArray", + "_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY", + "_Py_DECREF_NO_DEALLOC", + "_Py_DECREF_SPECIALIZED", + "_Py_EnterRecursiveCallTstateUnchecked", + "_Py_ID", + "_Py_IsImmortal", "_Py_LeaveRecursiveCallPy", - "CALL_STAT_INC", - "STAT_INC", + "_Py_LeaveRecursiveCallTstate", + "_Py_NewRef", + "_Py_SINGLETON", + "_Py_STR", + "_Py_atomic_load_uintptr_relaxed", + "_Py_set_eval_breaker_bit", + "advance_backoff_counter", + "assert", + "backoff_counter_triggers", + "initial_temperature_backoff_counter", "maybe_lltrace_resume_frame", - "_PyUnicode_JoinArray", - "_PyEval_FrameClearAndPop", - "_PyFrame_StackPush", - "PyCell_New", - "PyFloat_AS_DOUBLE", - "_PyFrame_PushUnchecked", - "Py_FatalError", - "STACKREFS_TO_PYOBJECTS", - "STACKREFS_TO_PYOBJECTS_CLEANUP", - "CONVERSION_FAILED", - "_PyList_FromStackRefSteal", - "_PyTuple_FromArraySteal", - "_PyTuple_FromStackRefSteal", -) - -ESCAPING_FUNCTIONS = ( - "import_name", - "import_from", + "restart_backoff_counter", ) - -def makes_escaping_api_call(instr: parser.InstDef) -> bool: - if "CALL_INTRINSIC" in instr.name: - return True - if instr.name == "_BINARY_OP": - return True - tkns = iter(instr.tokens) - for tkn in tkns: - if tkn.kind != lexer.IDENTIFIER: - continue +def find_stmt_start(node: parser.InstDef, idx: int) -> lexer.Token: + assert idx < len(node.block.tokens) + while True: + tkn = node.block.tokens[idx-1] + if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + break + idx -= 1 + assert idx > 0 + while node.block.tokens[idx].kind == "COMMENT": + idx += 1 + return node.block.tokens[idx] + + +def find_stmt_end(node: parser.InstDef, idx: int) -> lexer.Token: + assert idx < len(node.block.tokens) + while True: + idx += 1 + tkn = node.block.tokens[idx] + if tkn.kind == "SEMI": + return node.block.tokens[idx+1] + +def check_escaping_calls(instr: parser.InstDef, escapes: dict[lexer.Token, tuple[lexer.Token, lexer.Token]]) -> None: + calls = {escapes[t][0] for t in escapes} + in_if = 0 + tkn_iter = iter(instr.block.tokens) + for tkn in tkn_iter: + if tkn.kind == "IF": + next(tkn_iter) + in_if = 1 + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF"): + next(tkn_iter) + in_if = 1 + elif tkn.kind == "LPAREN" and in_if: + in_if += 1 + elif tkn.kind == "RPAREN": + if in_if: + in_if -= 1 + elif tkn in calls and in_if: + raise analysis_error(f"Escaping call '{tkn.text} in condition", tkn) + +def find_escaping_api_calls(instr: parser.InstDef) -> dict[lexer.Token, tuple[lexer.Token, lexer.Token]]: + result: dict[lexer.Token, tuple[lexer.Token, lexer.Token]] = {} + tokens = instr.block.tokens + for idx, tkn in enumerate(tokens): try: - next_tkn = next(tkns) - except StopIteration: - return False + next_tkn = tokens[idx+1] + except IndexError: + break + if tkn.kind == "SWITCH": + raise analysis_error(f"switch statements are not supported due to their complex flow control. Sorry.", tkn) if next_tkn.kind != lexer.LPAREN: continue - if tkn.text in ESCAPING_FUNCTIONS: - return True - if tkn.text == "tp_vectorcall": - return True - if not tkn.text.startswith("Py") and not tkn.text.startswith("_Py"): - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): + if tkn.kind == lexer.IDENTIFIER: + if tkn.text.upper() == tkn.text: + # simple macro + continue + #if not tkn.text.startswith(("Py", "_Py", "monitor")): + # continue + if tkn.text.startswith(("sym_", "optimize_")): + # Optimize functions + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + elif tkn.kind == "RPAREN": + prev = tokens[idx-1] + if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": + #cast + continue + elif tkn.kind != "RBRACKET": continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - return True - return False + start = find_stmt_start(instr, idx) + end = find_stmt_end(instr, idx) + result[start] = tkn, end + check_escaping_calls(instr, result) + return result EXITS = { @@ -637,6 +783,7 @@ def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: def compute_properties(op: parser.InstDef) -> Properties: + escaping_calls = find_escaping_api_calls(op) has_free = ( variable_used(op, "PyCell_New") or variable_used(op, "PyCell_GetRef") @@ -657,7 +804,7 @@ def compute_properties(op: parser.InstDef) -> Properties: error_with_pop = has_error_with_pop(op) error_without_pop = has_error_without_pop(op) return Properties( - escapes=makes_escaping_api_call(op), + escaping_calls=escaping_calls, error_with_pop=error_with_pop, error_without_pop=error_without_pop, deopts=deopts_if, @@ -692,6 +839,7 @@ def make_uop( stack=analyze_stack(op), caches=analyze_caches(inputs), deferred_refs=analyze_deferred_refs(op), + output_stores=find_stores_outputs(op), body=op.block.tokens, properties=compute_properties(op), ) @@ -712,6 +860,7 @@ def make_uop( stack=analyze_stack(op, bit), caches=analyze_caches(inputs), deferred_refs=analyze_deferred_refs(op), + output_stores=find_stores_outputs(op), body=op.block.tokens, properties=properties, ) @@ -735,6 +884,7 @@ def make_uop( stack=analyze_stack(op), caches=analyze_caches(inputs), deferred_refs=analyze_deferred_refs(op), + output_stores=find_stores_outputs(op), body=op.block.tokens, properties=properties, ) @@ -840,6 +990,7 @@ def add_pseudo( pseudo.name, analyze_stack(pseudo), [instructions[target] for target in pseudo.targets], + pseudo.as_sequence, pseudo.flags, ) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 069f0177a74018..8cba9127fa3071 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -18,8 +18,9 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): def set_position(self, tkn: Token) -> None: if self.last_token is not None: - if self.last_token.line < tkn.line: + if self.last_token.end_line < tkn.line: self.out.write("\n") + if self.last_token.line < tkn.line: if self.line_directives: self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') self.out.write(" " * self.indents[-1]) @@ -91,6 +92,8 @@ def emit_token(self, tkn: Token) -> None: self.maybe_dedent(tkn.text) self.set_position(tkn) self.emit_text(tkn.text) + if tkn.kind == "CMACRO": + self.newline = True self.maybe_indent(tkn.text) def emit_str(self, txt: str) -> None: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2f8fccec2ea409..7e032c21d2485c 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -9,18 +9,47 @@ analysis_error, ) from cwriter import CWriter -from typing import Callable, Mapping, TextIO, Iterator +from typing import Callable, Mapping, TextIO, Iterator, Iterable from lexer import Token -from stack import Stack +from stack import Stack, Local, Storage, StackError +# Set this to true for voluminous output showing state of stack and locals +PRINT_STACKS = False -ROOT = Path(__file__).parent.parent.parent -DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix() +class TokenIterator: + + look_ahead: Token | None + iterator: Iterator[Token] + + def __init__(self, tkns: Iterable[Token]): + self.iterator = iter(tkns) + self.look_ahead = None + + def __iter__(self) -> "TokenIterator": + return self + + def __next__(self) -> Token: + if self.look_ahead is None: + return next(self.iterator) + else: + res = self.look_ahead + self.look_ahead = None + return res + + def peek(self) -> Token | None: + if self.look_ahead is None: + for tkn in self.iterator: + self.look_ahead = tkn + break + return self.look_ahead + +ROOT = Path(__file__).parent.parent.parent.resolve() +DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").as_posix() def root_relative_path(filename: str) -> str: try: - return Path(filename).absolute().relative_to(ROOT).as_posix() + return Path(filename).resolve().relative_to(ROOT).as_posix() except ValueError: # Not relative to root, just return original path. return filename @@ -47,22 +76,28 @@ def write_header( ) -def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: +def emit_to(out: CWriter, tkn_iter: TokenIterator, end: str) -> Token: parens = 0 for tkn in tkn_iter: if tkn.kind == end and parens == 0: - return + return tkn if tkn.kind == "LPAREN": parens += 1 if tkn.kind == "RPAREN": parens -= 1 out.emit(tkn) + raise analysis_error(f"Expecting {end}. Reached end of file", tkn) ReplacementFunctionType = Callable[ - [Token, Iterator[Token], Uop, Stack, Instruction | None], None + [Token, TokenIterator, Uop, Storage, Instruction | None], bool ] +def always_true(tkn: Token | None) -> bool: + if tkn is None: + return False + return tkn.text in {"true", "1"} + class Emitter: out: CWriter @@ -75,21 +110,42 @@ def __init__(self, out: CWriter): "ERROR_IF": self.error_if, "ERROR_NO_POP": self.error_no_pop, "DECREF_INPUTS": self.decref_inputs, + "DEAD": self.kill, + "INPUTS_DEAD": self.kill_inputs, "SYNC_SP": self.sync_sp, - "PyStackRef_FromPyObjectNew": self.py_stack_ref_from_py_object_new, + "SAVE_STACK": self.save_stack, + "RELOAD_STACK": self.reload_stack, + "PyStackRef_CLOSE": self.stackref_close, + "PyStackRef_CLOSE_SPECIALIZED": self.stackref_close, + "PyStackRef_AsPyObjectSteal": self.stackref_steal, + "DISPATCH": self.dispatch } self.out = out + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.emit(tkn) + return False + def deopt_if( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - unused: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: self.out.emit_at("DEOPT_IF", tkn) - self.out.emit(next(tkn_iter)) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() emit_to(self.out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon self.out.emit(", ") @@ -97,25 +153,30 @@ def deopt_if( assert inst.family is not None self.out.emit(inst.family.name) self.out.emit(");\n") + return not always_true(first_tkn) exit_if = deopt_if def error_if( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: self.out.emit_at("if ", tkn) - self.out.emit(next(tkn_iter)) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() emit_to(self.out, tkn_iter, "COMMA") label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon self.out.emit(") ") - c_offset = stack.peek_offset() + storage.clear_inputs("at ERROR_IF") + c_offset = storage.stack.peek_offset() try: offset = -int(c_offset) except ValueError: @@ -130,33 +191,35 @@ def error_if( self.out.emit(";\n") else: self.out.emit("{\n") - stack.flush_locally(self.out) + storage.copy().flush(self.out) self.out.emit("goto ") self.out.emit(label) self.out.emit(";\n") self.out.emit("}\n") + return not always_true(first_tkn) def error_no_pop( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: next(tkn_iter) # LPAREN next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon self.out.emit_at("goto error;", tkn) + return False def decref_inputs( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -178,58 +241,278 @@ def decref_inputs( self.out.emit(f"PyStackRef_XCLOSE({var.name});\n") else: self.out.emit(f"PyStackRef_CLOSE({var.name});\n") + for input in storage.inputs: + input.defined = False + return True + + def kill_inputs( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + for var in storage.inputs: + var.defined = False + return True + + def kill( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + name_tkn = next(tkn_iter) + name = name_tkn.text + next(tkn_iter) + next(tkn_iter) + for var in storage.inputs: + if var.name == name: + var.defined = False + break + else: + raise analysis_error(f"'{name}' is not a live input-only variable", name_tkn) + return True + + def stackref_close( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + tkn = next(tkn_iter) + assert tkn.kind == "LPAREN" + self.out.emit(tkn) + name = next(tkn_iter) + self.out.emit(name) + if name.kind == "IDENTIFIER": + for var in storage.inputs: + if var.name == name.text: + var.defined = False + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + return True + + stackref_steal = stackref_close def sync_sp( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: next(tkn_iter) next(tkn_iter) next(tkn_iter) - stack.flush(self.out) + storage.clear_inputs("when syncing stack") + storage.flush(self.out) + self._print_storage(storage) + return True + + def emit_save(self, storage: Storage) -> None: + storage.save(self.out) + self._print_storage(storage) - def py_stack_ref_from_py_object_new( + def save_stack( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.emit_save(storage) + return True + + def emit_reload(self, storage: Storage) -> None: + storage.reload(self.out) + self._print_storage(storage) + + def reload_stack( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + self.emit_reload(storage) + return True + + def _print_storage(self, storage: Storage) -> None: + if PRINT_STACKS: + self.out.start_line() + self.emit(storage.as_comment()) + self.out.start_line() + + def _emit_if( + self, + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> tuple[bool, Token, Storage]: + """Returns (reachable?, closing '}', stack).""" + tkn = next(tkn_iter) + assert tkn.kind == "LPAREN" self.out.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - self.out.emit(";\n") + rparen = emit_to(self.out, tkn_iter, "RPAREN") + self.emit(rparen) + if_storage = storage.copy() + reachable, rbrace, if_storage = self._emit_block(tkn_iter, uop, if_storage, inst, True) + try: + maybe_else = tkn_iter.peek() + if maybe_else and maybe_else.kind == "ELSE": + self._print_storage(storage) + self.emit(rbrace) + self.emit(next(tkn_iter)) + maybe_if = tkn_iter.peek() + if maybe_if and maybe_if.kind == "IF": + #Emit extra braces around the if to get scoping right + self.emit(" {\n") + self.emit(next(tkn_iter)) + else_reachable, rbrace, else_storage = self._emit_if(tkn_iter, uop, storage, inst) + self.out.start_line() + self.emit("}\n") + else: + else_reachable, rbrace, else_storage = self._emit_block(tkn_iter, uop, storage, inst, True) + if not reachable: + # Discard the if storage + reachable = else_reachable + storage = else_storage + elif not else_reachable: + # Discard the else storage + storage = if_storage + reachable = True + else: + if PRINT_STACKS: + self.emit("/* Merge */\n") + else_storage.merge(if_storage, self.out) + storage = else_storage + self._print_storage(storage) + else: + if reachable: + if PRINT_STACKS: + self.emit("/* Merge */\n") + if_storage.merge(storage, self.out) + storage = if_storage + self._print_storage(storage) + else: + # Discard the if storage + reachable = True + except StackError as ex: + self._print_storage(if_storage) + raise analysis_error(ex.args[0], rbrace) # from None + return reachable, rbrace, storage - target = uop.deferred_refs[tkn] - if target is None: - # An assignment we don't handle, such as to a pointer or array. - return + def _emit_block( + self, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + emit_first_brace: bool + ) -> tuple[bool, Token, Storage]: + """ Returns (reachable?, closing '}', stack).""" + braces = 1 + out_stores = set(uop.output_stores) + tkn = next(tkn_iter) + reload: Token | None = None + try: + reachable = True + line : int = -1 + if tkn.kind != "LBRACE": + raise analysis_error(f"PEP 7: expected '{{', found: {tkn.text}", tkn) + escaping_calls = uop.properties.escaping_calls + if emit_first_brace: + self.emit(tkn) + self._print_storage(storage) + for tkn in tkn_iter: + if PRINT_STACKS and tkn.line != line: + self.out.start_line() + self.emit(storage.as_comment()) + self.out.start_line() + line = tkn.line + if tkn in escaping_calls: + if tkn != reload: + self.emit_save(storage) + _, reload = escaping_calls[tkn] + elif tkn == reload: + self.emit_reload(storage) + if tkn.kind == "LBRACE": + self.out.emit(tkn) + braces += 1 + elif tkn.kind == "RBRACE": + self._print_storage(storage) + braces -= 1 + if braces == 0: + return reachable, tkn, storage + self.out.emit(tkn) + elif tkn.kind == "GOTO": + reachable = False; + self.out.emit(tkn) + elif tkn.kind == "IDENTIFIER": + if tkn.text in self._replacers: + if not self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst): + reachable = False + else: + if tkn in out_stores: + for out in storage.outputs: + if out.name == tkn.text: + out.defined = True + out.in_memory = False + break + if tkn.text.startswith("DISPATCH"): + self._print_storage(storage) + reachable = False + self.out.emit(tkn) + elif tkn.kind == "IF": + self.out.emit(tkn) + if_reachable, rbrace, storage = self._emit_if(tkn_iter, uop, storage, inst) + if reachable: + reachable = if_reachable + self.out.emit(rbrace) + else: + self.out.emit(tkn) + except StackError as ex: + raise analysis_error(ex.args[0], tkn) from None + raise analysis_error("Expecting closing brace. Reached end of file", tkn) - # Flush the assignment to the stack. Note that we don't flush the - # stack pointer here, and instead are currently relying on initializing - # unused portions of the stack to NULL. - stack.flush_single_var(self.out, target, uop.stack.outputs) def emit_tokens( self, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: - tkns = uop.body[1:-1] - if not tkns: - return - tkn_iter = iter(tkns) + ) -> Storage: + tkn_iter = TokenIterator(uop.body) self.out.start_line() - for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in self._replacers: - self._replacers[tkn.text](tkn, tkn_iter, uop, stack, inst) - else: - self.out.emit(tkn) + _, rbrace, storage = self._emit_block(tkn_iter, uop, storage, inst, False) + try: + self._print_storage(storage) + storage.push_outputs() + self._print_storage(storage) + except StackError as ex: + raise analysis_error(ex.args[0], rbrace) + return storage def emit(self, txt: str | Token) -> None: self.out.emit(txt) diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index ba09931c541646..6cf36f343d5fa7 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -74,7 +74,7 @@ We update it as the need arises. ### Syntax Each op definition has a kind, a name, a stack and instruction stream effect, -and a piece of C code describing its semantics:: +and a piece of C code describing its semantics: ``` file: @@ -245,7 +245,8 @@ The same is true for all members of a pseudo instruction ## Examples -(Another source of examples can be found in the [tests](test_generator.py).) +(Another source of examples can be found in the +[tests](https://github.com/python/cpython/blob/main/Lib/test/test_generated_cases.py).) Some examples: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index d5831593215f76..37f96398ff175f 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -79,7 +79,7 @@ def choice(*opts: str) -> str: opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" +macro = r"#.*\n" CMACRO = "CMACRO" id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" @@ -333,6 +333,9 @@ def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: line += newlines else: begin = line, start - linestart + if kind == CMACRO: + linestart = end + line += 1 if kind != "\n": yield Token( filename, kind, text, begin, (line, start - linestart + len(text)) diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9b1bc98b5c08d7..2ad7604af9cc0d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -305,6 +305,7 @@ def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: table_size = len(analysis.pseudos) max_targets = max(len(pseudo.targets) for pseudo in analysis.pseudos.values()) out.emit("struct pseudo_targets {\n") + out.emit(f"uint8_t as_sequence;\n") out.emit(f"uint8_t targets[{max_targets + 1}];\n") out.emit("};\n") out.emit( @@ -315,10 +316,11 @@ def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: f"const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}] = {{\n" ) for pseudo in analysis.pseudos.values(): + as_sequence = "1" if pseudo.as_sequence else "0" targets = ["0"] * (max_targets + 1) for i, target in enumerate(pseudo.targets): targets[i] = target.name - out.emit(f"[{pseudo.name}-256] = {{ {{ {', '.join(targets)} }} }},\n") + out.emit(f"[{pseudo.name}-256] = {{ {as_sequence}, {{ {', '.join(targets)} }} }},\n") out.emit("};\n\n") out.emit("#endif // NEED_OPCODE_METADATA\n") out.emit("static inline bool\n") diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index b74f627235ad84..7a1dfe1b85bf1a 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -18,11 +18,12 @@ ROOT, write_header, Emitter, + TokenIterator, ) from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import Local, Stack, StackError +from stack import Local, Stack, StackError, Storage DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() @@ -45,7 +46,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: variables = {"unused"} if not skip_inputs: for var in reversed(uop.stack.inputs): - if var.name not in variables: + if var.used and var.name not in variables: variables.add(var.name) if var.condition: out.emit(f"{type_name(var)}{var.name} = NULL;\n") @@ -65,7 +66,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: def decref_inputs( out: CWriter, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, stack: Stack, inst: Instruction | None, @@ -76,13 +77,27 @@ def decref_inputs( out.emit_at("", tkn) -def emit_default(out: CWriter, uop: Uop) -> None: - for i, var in enumerate(uop.stack.outputs): +def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: + for var in reversed(uop.stack.inputs): + stack.pop(var) + top_offset = stack.top_offset.copy() + for var in uop.stack.outputs: + if var.is_array() and not var.peek and not var.name == "unused": + c_offset = top_offset.to_c() + out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n") + top_offset.push(var) + for var in uop.stack.outputs: + local = Local.undefined(var) + stack.push(local) if var.name != "unused" and not var.peek: + local.defined = True if var.is_array(): - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") - out.emit("}\n") + if var.size == "1": + out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") + else: + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit("}\n") elif var.name == "null": out.emit(f"{var.name} = sym_new_null(ctx);\n") else: @@ -90,7 +105,12 @@ def emit_default(out: CWriter, uop: Uop) -> None: class OptimizerEmitter(Emitter): - pass + + def emit_save(self, storage: Storage) -> None: + storage.flush(self.out) + + def emit_reload(self, storage: Storage) -> None: + pass def write_uop( @@ -102,22 +122,18 @@ def write_uop( skip_inputs: bool, ) -> None: locals: dict[str, Local] = {} + prototype = override if override else uop try: - prototype = override if override else uop - is_override = override is not None out.start_line() - for var in reversed(prototype.stack.inputs): - code, local = stack.pop(var, extract_bits=True) - if not skip_inputs: + if override: + code_list, storage = Storage.for_uop(stack, prototype, extract_bits=False) + for code in code_list: out.emit(code) - if local.defined: - locals[local.name] = local - out.emit(stack.define_output_arrays(prototype.stack.outputs)) if debug: args = [] - for var in prototype.stack.inputs: - if not var.peek or is_override: - args.append(var.name) + for input in prototype.stack.inputs: + if not input.peek or override: + args.append(input.name) out.emit(f'DEBUG_PRINTF({", ".join(args)});\n') if override: for cache in uop.caches: @@ -130,20 +146,18 @@ def write_uop( out.emit(f"{type}{cache.name} = ({cast})this_instr->operand;\n") if override: emitter = OptimizerEmitter(out) - emitter.emit_tokens(override, stack, None) + # No reference management of inputs needed. + for var in storage.inputs: # type: ignore[possibly-undefined] + var.defined = False + storage = emitter.emit_tokens(override, storage, None) + out.start_line() + storage.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=False) else: - emit_default(out, uop) - - for var in prototype.stack.outputs: - if var.name in locals: - local = locals[var.name] - else: - local = Local.local(var) - stack.push(local) - out.start_line() - stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) + emit_default(out, uop, stack) + out.start_line() + stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=False) except StackError as ex: - raise analysis_error(ex.args[0], uop.body[0]) + raise analysis_error(ex.args[0], prototype.body[0]) # from None SKIPS = ("_EXTENDED_ARG",) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index ab5444d41ac6a9..de31d9b232f9df 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -148,6 +148,7 @@ class Pseudo(Node): outputs: list[OutputEffect] flags: list[str] # instr flags to set on the pseudo instruction targets: list[str] # opcodes this can be replaced by + as_sequence: bool AstNode = InstDef | Macro | Pseudo | Family @@ -423,16 +424,22 @@ def pseudo_def(self) -> Pseudo | None: flags = [] if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): - if not self.expect(lx.LBRACE): - raise self.make_syntax_error("Expected {") - if members := self.members(): - if self.expect(lx.RBRACE) and self.expect(lx.SEMI): + if self.expect(lx.LBRACE): + as_sequence = False + closing = lx.RBRACE + elif self.expect(lx.LBRACKET): + as_sequence = True + closing = lx.RBRACKET + else: + raise self.make_syntax_error("Expected { or [") + if members := self.members(allow_sequence=True): + if self.expect(closing) and self.expect(lx.SEMI): return Pseudo( - tkn.text, inp, outp, flags, members + tkn.text, inp, outp, flags, members, as_sequence ) return None - def members(self) -> list[str] | None: + def members(self, allow_sequence : bool=False) -> list[str] | None: here = self.getpos() if tkn := self.expect(lx.IDENTIFIER): members = [tkn.text] @@ -442,8 +449,10 @@ def members(self) -> list[str] | None: else: break peek = self.peek() - if not peek or peek.kind != lx.RBRACE: - raise self.make_syntax_error("Expected comma or right paren") + kinds = [lx.RBRACE, lx.RBRACKET] if allow_sequence else [lx.RBRACE] + if not peek or peek.kind not in kinds: + raise self.make_syntax_error( + f"Expected comma or right paren{'/bracket' if allow_sequence else ''}") return members self.setpos(here) return None diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index de4d900563ee0b..a954bed4df073c 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -46,20 +46,41 @@ class Local: in_memory: bool defined: bool + def __repr__(self) -> str: + return f"Local('{self.item.name}', mem={self.in_memory}, defined={self.defined}, array={self.is_array()})" + + def compact_str(self) -> str: + mtag = "M" if self.in_memory else "" + dtag = "D" if self.defined else "" + atag = "A" if self.is_array() else "" + return f"'{self.item.name}'{mtag}{dtag}{atag}" + @staticmethod def unused(defn: StackItem) -> "Local": return Local(defn, False, defn.is_array(), False) @staticmethod - def local(defn: StackItem) -> "Local": + def undefined(defn: StackItem) -> "Local": array = defn.is_array() - return Local(defn, not array, array, True) + return Local(defn, not array, array, False) @staticmethod def redefinition(var: StackItem, prev: "Local") -> "Local": assert var.is_array() == prev.is_array() return Local(var, prev.cached, prev.in_memory, True) + @staticmethod + def from_memory(defn: StackItem) -> "Local": + return Local(defn, True, True, True) + + def copy(self) -> "Local": + return Local( + self.item, + self.cached, + self.in_memory, + self.defined + ) + @property def size(self) -> str: return self.item.size @@ -75,6 +96,16 @@ def condition(self) -> str | None: def is_array(self) -> bool: return self.item.is_array() + def __eq__(self, other: object) -> bool: + if not isinstance(other, Local): + return NotImplemented + return ( + self.item is other.item + and self.cached is other.cached + and self.in_memory is other.in_memory + and self.defined is other.defined + ) + @dataclass class StackOffset: @@ -156,10 +187,34 @@ def to_c(self) -> str: res = "-" + res[3:] return res + def as_int(self) -> int | None: + self.simplify() + int_offset = 0 + for item in self.popped: + try: + int_offset -= int(item) + except ValueError: + return None + for item in self.pushed: + try: + int_offset += int(item) + except ValueError: + return None + return int_offset + def clear(self) -> None: self.popped = [] self.pushed = [] + def __bool__(self) -> bool: + self.simplify() + return bool(self.popped) or bool(self.pushed) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, StackOffset): + return NotImplemented + return self.to_c() == other.to_c() + class StackError(Exception): pass @@ -174,7 +229,7 @@ def __init__(self) -> None: self.variables: list[Local] = [] self.defined: set[str] = set() - def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: + def pop(self, var: StackItem, extract_bits: bool = True) -> tuple[str, Local]: self.top_offset.pop(var) indirect = "&" if var.is_array() else "" if self.variables: @@ -192,7 +247,7 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: if var.name in UNUSED: if popped.name not in UNUSED and popped.name in self.defined: raise StackError( - f"Value is declared unused, but is already cached by prior operation" + f"Value is declared unused, but is already cached by prior operation as '{popped.name}'" ) return "", popped if not var.used: @@ -208,6 +263,7 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n" else: defn = f"{var.name} = stack_pointer[{self.top_offset.to_c()}];\n" + popped.in_memory = True return defn, Local.redefinition(var, popped) self.base_offset.pop(var) @@ -215,7 +271,7 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: return "", Local.unused(var) self.defined.add(var.name) cast = f"({var.type})" if (not indirect and var.type) else "" - bits = ".bits" if cast and not extract_bits else "" + bits = ".bits" if cast and extract_bits else "" assign = f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" if var.condition: if var.condition == "1": @@ -226,27 +282,14 @@ def pop(self, var: StackItem, extract_bits: bool = False) -> tuple[str, Local]: assign = f"if ({var.condition}) {{ {assign} }}\n" else: assign = f"{assign}\n" - in_memory = var.is_array() or var.peek - return assign, Local(var, not var.is_array(), in_memory, True) + return assign, Local.from_memory(var) def push(self, var: Local) -> None: + assert(var not in self.variables) self.variables.append(var) self.top_offset.push(var.item) if var.item.used: self.defined.add(var.name) - var.defined = True - - def define_output_arrays(self, outputs: list[StackItem]) -> str: - res = [] - top_offset = self.top_offset.copy() - for var in outputs: - if var.is_array() and var.used and not var.peek: - c_offset = top_offset.to_c() - top_offset.push(var) - res.append(f"{var.name} = &stack_pointer[{c_offset}];\n") - else: - top_offset.push(var) - return "\n".join(res) @staticmethod def _do_emit( @@ -254,102 +297,92 @@ def _do_emit( var: StackItem, base_offset: StackOffset, cast_type: str = "uintptr_t", - extract_bits: bool = False, + extract_bits: bool = True, ) -> None: cast = f"({cast_type})" if var.type else "" - bits = ".bits" if cast and not extract_bits else "" + bits = ".bits" if cast and extract_bits else "" if var.condition == "0": return if var.condition and var.condition != "1": out.emit(f"if ({var.condition}) ") out.emit(f"stack_pointer[{base_offset.to_c()}]{bits} = {cast}{var.name};\n") - @staticmethod - def _do_flush( - out: CWriter, - variables: list[Local], - base_offset: StackOffset, - top_offset: StackOffset, - cast_type: str = "uintptr_t", - extract_bits: bool = False, - ) -> None: - out.start_line() - for var in variables: - if ( - var.cached - and not var.in_memory - and not var.item.peek - and not var.name in UNUSED - ): - Stack._do_emit(out, var.item, base_offset, cast_type, extract_bits) - base_offset.push(var.item) - if base_offset.to_c() != top_offset.to_c(): - print("base", base_offset, "top", top_offset) - assert False - number = base_offset.to_c() + def _adjust_stack_pointer(self, out: CWriter, number: str) -> None: if number != "0": + out.start_line() out.emit(f"stack_pointer += {number};\n") out.emit("assert(WITHIN_STACK_BOUNDS());\n") - out.start_line() - - def flush_locally( - self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False - ) -> None: - self._do_flush( - out, - self.variables[:], - self.base_offset.copy(), - self.top_offset.copy(), - cast_type, - extract_bits, - ) def flush( - self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False + self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True ) -> None: - self._do_flush( - out, - self.variables, - self.base_offset, - self.top_offset, - cast_type, - extract_bits, - ) - self.variables = [] - self.base_offset.clear() + out.start_line() + var_offset = self.base_offset.copy() + for var in self.variables: + if ( + var.defined and + not var.in_memory + ): + Stack._do_emit(out, var.item, var_offset, cast_type, extract_bits) + var.in_memory = True + var_offset.push(var.item) + number = self.top_offset.to_c() + self._adjust_stack_pointer(out, number) + self.base_offset -= self.top_offset self.top_offset.clear() + out.start_line() - def flush_single_var( - self, - out: CWriter, - var_name: str, - outputs: list[StackItem], - cast_type: str = "uintptr_t", - extract_bits: bool = False, - ) -> None: - assert any(var.name == var_name for var in outputs) - base_offset = self.base_offset.copy() - top_offset = self.top_offset.copy() - for var in self.variables: - base_offset.push(var.item) - for output in outputs: - if any(output == v.item for v in self.variables): - # The variable is already on the stack, such as a peeked value - # in the tier1 generator - continue - if output.name == var_name: - Stack._do_emit(out, output, base_offset, cast_type, extract_bits) - base_offset.push(output) - top_offset.push(output) - if base_offset.to_c() != top_offset.to_c(): - print("base", base_offset, "top", top_offset) - assert False + def is_flushed(self) -> bool: + return not self.variables and not self.base_offset and not self.top_offset def peek_offset(self) -> str: return self.top_offset.to_c() def as_comment(self) -> str: - return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + variables = ", ".join([v.compact_str() for v in self.variables]) + return ( + f"/* Variables: {variables}. base: {self.base_offset.to_c()}. top: {self.top_offset.to_c()} */" + ) + + def copy(self) -> "Stack": + other = Stack() + other.top_offset = self.top_offset.copy() + other.base_offset = self.base_offset.copy() + other.variables = [var.copy() for var in self.variables] + other.defined = set(self.defined) + return other + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Stack): + return NotImplemented + return ( + self.top_offset == other.top_offset + and self.base_offset == other.base_offset + and self.variables == other.variables + ) + + def align(self, other: "Stack", out: CWriter) -> None: + if len(self.variables) != len(other.variables): + raise StackError("Cannot align stacks: differing variables") + if self.top_offset == other.top_offset: + return + diff = self.top_offset - other.top_offset + try: + self.top_offset -= diff + self.base_offset -= diff + self._adjust_stack_pointer(out, diff.to_c()) + except ValueError: + raise StackError("Cannot align stacks: cannot adjust stack pointer") + + def merge(self, other: "Stack", out: CWriter) -> None: + if len(self.variables) != len(other.variables): + raise StackError("Cannot merge stacks: differing variables") + for self_var, other_var in zip(self.variables, other.variables): + if self_var.name != other_var.name: + raise StackError(f"Mismatched variables on stack: {self_var.name} and {other_var.name}") + self_var.defined = self_var.defined and other_var.defined + self_var.in_memory = self_var.in_memory and other_var.in_memory + self.align(other, out) def get_stack_effect(inst: Instruction | PseudoInstruction) -> Stack: @@ -377,3 +410,213 @@ def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: local = Local.unused(var) stack.push(local) return stack + +@dataclass +class Storage: + + stack: Stack + inputs: list[Local] + outputs: list[Local] + peeks: list[Local] + spilled: int = 0 + + @staticmethod + def needs_defining(var: Local) -> bool: + return ( + not var.defined and + not var.is_array() and + var.name != "unused" + ) + + @staticmethod + def is_live(var: Local) -> bool: + return ( + var.defined and + var.name != "unused" + ) + + def first_input_not_cleared(self) -> str: + for input in self.inputs: + if input.defined: + return input.name + return "" + + def clear_inputs(self, reason:str) -> None: + while self.inputs: + tos = self.inputs.pop() + if self.is_live(tos) and not tos.is_array(): + raise StackError( + f"Input '{tos.name}' is still live {reason}" + ) + self.stack.pop(tos.item) + + def clear_dead_inputs(self) -> None: + live = "" + while self.inputs: + tos = self.inputs[-1] + if self.is_live(tos): + live = tos.name + break + self.inputs.pop() + self.stack.pop(tos.item) + for var in self.inputs: + if not var.defined and not var.is_array() and var.name != "unused": + raise StackError( + f"Input '{var.name}' is not live, but '{live}' is" + ) + + def _push_defined_outputs(self) -> None: + defined_output = "" + for output in self.outputs: + if output.defined and not output.in_memory: + defined_output = output.name + if not defined_output: + return + self.clear_inputs(f"when output '{defined_output}' is defined") + undefined = "" + for out in self.outputs: + if out.defined: + if undefined: + f"Locals not defined in stack order. " + f"Expected '{undefined}' to be defined before '{out.name}'" + else: + undefined = out.name + while self.outputs and not self.needs_defining(self.outputs[0]): + out = self.outputs.pop(0) + self.stack.push(out) + + def locals_cached(self) -> bool: + for out in self.outputs: + if out.defined: + return True + return False + + def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True) -> None: + self.clear_dead_inputs() + self._push_defined_outputs() + self.stack.flush(out, cast_type, extract_bits) + + def save(self, out: CWriter) -> None: + assert self.spilled >= 0 + if self.spilled == 0: + self.flush(out) + out.start_line() + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + self.spilled += 1 + + def reload(self, out: CWriter) -> None: + if self.spilled == 0: + raise StackError("Cannot reload stack as it hasn't been saved") + assert self.spilled > 0 + self.spilled -= 1 + if self.spilled == 0: + out.start_line() + out.emit("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + + @staticmethod + def for_uop(stack: Stack, uop: Uop, extract_bits: bool = True) -> tuple[list[str], "Storage"]: + code_list: list[str] = [] + inputs: list[Local] = [] + peeks: list[Local] = [] + for input in reversed(uop.stack.inputs): + code, local = stack.pop(input, extract_bits) + code_list.append(code) + if input.peek: + peeks.append(local) + else: + inputs.append(local) + inputs.reverse() + peeks.reverse() + for peek in peeks: + stack.push(peek) + top_offset = stack.top_offset.copy() + for ouput in uop.stack.outputs: + if ouput.is_array() and ouput.used and not ouput.peek: + c_offset = top_offset.to_c() + top_offset.push(ouput) + code_list.append(f"{ouput.name} = &stack_pointer[{c_offset}];\n") + else: + top_offset.push(ouput) + for var in inputs: + stack.push(var) + outputs = [ Local.undefined(var) for var in uop.stack.outputs if not var.peek ] + return code_list, Storage(stack, inputs, outputs, peeks) + + @staticmethod + def copy_list(arg: list[Local]) -> list[Local]: + return [ l.copy() for l in arg ] + + def copy(self) -> "Storage": + new_stack = self.stack.copy() + variables = { var.name: var for var in new_stack.variables } + inputs = [ variables[var.name] for var in self.inputs] + assert [v.name for v in inputs] == [v.name for v in self.inputs], (inputs, self.inputs) + return Storage( + new_stack, inputs, + self.copy_list(self.outputs), self.copy_list(self.peeks) + ) + + def sanity_check(self) -> None: + names: set[str] = set() + for var in self.inputs: + if var.name in names: + raise StackError(f"Duplicate name {var.name}") + names.add(var.name) + names = set() + for var in self.outputs: + if var.name in names: + raise StackError(f"Duplicate name {var.name}") + names.add(var.name) + names = set() + for var in self.stack.variables: + if var.name in names: + raise StackError(f"Duplicate name {var.name}") + names.add(var.name) + + def is_flushed(self) -> bool: + for var in self.outputs: + if var.defined and not var.in_memory: + return False + return self.stack.is_flushed() + + def merge(self, other: "Storage", out: CWriter) -> None: + self.sanity_check() + if len(self.inputs) != len(other.inputs): + self.clear_dead_inputs() + other.clear_dead_inputs() + if len(self.inputs) != len(other.inputs): + diff = self.inputs[-1] if len(self.inputs) > len(other.inputs) else other.inputs[-1] + raise StackError(f"Unmergeable inputs. Differing state of '{diff.name}'") + for var, other_var in zip(self.inputs, other.inputs): + if var.defined != other_var.defined: + raise StackError(f"'{var.name}' is cleared on some paths, but not all") + if len(self.outputs) != len(other.outputs): + self._push_defined_outputs() + other._push_defined_outputs() + if len(self.outputs) != len(other.outputs): + var = self.outputs[0] if len(self.outputs) > len(other.outputs) else other.outputs[0] + raise StackError(f"'{var.name}' is set on some paths, but not all") + self.stack.merge(other.stack, out) + self.sanity_check() + + def push_outputs(self) -> None: + if self.spilled: + raise StackError(f"Unbalanced stack spills") + self.clear_inputs("at the end of the micro-op") + if self.inputs: + raise StackError(f"Input variable '{self.inputs[-1].name}' is still live") + self._push_defined_outputs() + if self.outputs: + for out in self.outputs: + if self.needs_defining(out): + raise StackError(f"Output variable '{self.outputs[0].name}' is not defined") + self.stack.push(out) + self.outputs = [] + + def as_comment(self) -> str: + stack_comment = self.stack.as_comment() + next_line = "\n " + inputs = ", ".join([var.compact_str() for var in self.inputs]) + outputs = ", ".join([var.compact_str() for var in self.outputs]) + peeks = ", ".join([var.name for var in self.peeks]) + return f"{stack_comment[:-2]}{next_line}inputs: {inputs}{next_line}outputs: {outputs}{next_line}peeks: {peeks} */" diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c749896c2cb7f6..8dadc5736c8889 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -22,10 +22,11 @@ write_header, type_and_null, Emitter, + TokenIterator, ) from cwriter import CWriter from typing import TextIO -from stack import Local, Stack, StackError, get_stack_effect +from stack import Local, Stack, StackError, get_stack_effect, Storage DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -47,7 +48,7 @@ def declare_variables(inst: Instruction, out: CWriter) -> None: try: stack = get_stack_effect(inst) except StackError as ex: - raise analysis_error(ex.args[0], inst.where) + raise analysis_error(ex.args[0], inst.where) from None required = set(stack.defined) required.discard("unused") for part in inst.parts: @@ -70,46 +71,26 @@ def write_uop( stack: Stack, inst: Instruction, braces: bool, -) -> int: +) -> tuple[int, Stack]: # out.emit(stack.as_comment() + "\n") if isinstance(uop, Skip): entries = "entries" if uop.size > 1 else "entry" emitter.emit(f"/* Skip {uop.size} cache {entries} */\n") - return offset + uop.size + return (offset + uop.size), stack if isinstance(uop, Flush): emitter.emit(f"// flush\n") stack.flush(emitter.out) - return offset + return offset, stack try: locals: dict[str, Local] = {} emitter.out.start_line() if braces: emitter.out.emit(f"// {uop.name}\n") - peeks: list[Local] = [] - for var in reversed(uop.stack.inputs): - code, local = stack.pop(var) - emitter.emit(code) - if var.peek: - peeks.append(local) - if local.defined: - locals[local.name] = local - # Push back the peeks, so that they remain on the logical - # stack, but their values are cached. - while peeks: - stack.push(peeks.pop()) - if braces: emitter.emit("{\n") - emitter.out.emit(stack.define_output_arrays(uop.stack.outputs)) - outputs: list[Local] = [] - for var in uop.stack.outputs: - if not var.peek: - if var.name in locals: - local = locals[var.name] - elif var.name == "unused": - local = Local.unused(var) - else: - local = Local.local(var) - outputs.append(local) + code_list, storage = Storage.for_uop(stack, uop) + emitter._print_storage(storage) + for code in code_list: + emitter.emit(code) for cache in uop.caches: if cache.name != "unused": @@ -125,17 +106,13 @@ def write_uop( if inst.family is None: emitter.emit(f"(void){cache.name};\n") offset += cache.size - emitter.emit_tokens(uop, stack, inst) - for output in outputs: - if output.name in uop.deferred_refs.values(): - # We've already spilled this when emitting tokens - output.cached = False - stack.push(output) + + storage = emitter.emit_tokens(uop, storage, inst) if braces: emitter.out.start_line() emitter.emit("}\n") # emitter.emit(stack.as_comment() + "\n") - return offset + return offset, storage.stack except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) @@ -173,9 +150,9 @@ def generate_tier1( out.emit(f"TARGET({name}) {{\n") unused_guard = "(void)this_instr;\n" if inst.family is None else "" if inst.properties.needs_prev: - out.emit(f"_Py_CODEUNIT *prev_instr = frame->instr_ptr;\n") + out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") if needs_this and not inst.is_target: - out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") out.emit(unused_guard) else: out.emit(f"frame->instr_ptr = next_instr;\n") @@ -184,7 +161,7 @@ def generate_tier1( if inst.is_target: out.emit(f"PREDICTED({name});\n") if needs_this: - out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) if inst.family is not None: out.emit( @@ -197,10 +174,11 @@ def generate_tier1( for part in inst.parts: # Only emit braces if more than one uop insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - offset = write_uop(part, emitter, offset, stack, inst, insert_braces) + offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) out.start_line() + + stack.flush(out) if not inst.parts[-1].properties.always_exits: - stack.flush(out) out.emit("DISPATCH();\n") out.start_line() out.emit("}") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index b7c70fdad085fd..634848c10309d5 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -20,11 +20,13 @@ write_header, type_and_null, Emitter, + TokenIterator, + always_true, ) from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import Local, Stack, StackError, get_stack_effect +from stack import Local, Stack, StackError, Storage DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" @@ -32,7 +34,7 @@ def declare_variable( var: StackItem, uop: Uop, required: set[str], out: CWriter ) -> None: - if var.name not in required: + if not var.used or var.name not in required: return required.remove(var.name) type, null = type_and_null(var) @@ -52,7 +54,7 @@ def declare_variables(uop: Uop, out: CWriter) -> None: for var in reversed(uop.stack.inputs): stack.pop(var) for var in uop.stack.outputs: - stack.push(Local.unused(var)) + stack.push(Local.undefined(var)) required = set(stack.defined) required.discard("unused") for var in reversed(uop.stack.inputs): @@ -69,88 +71,103 @@ def __init__(self, out: CWriter): def error_if( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: self.out.emit_at("if ", tkn) - self.emit(next(tkn_iter)) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = next(tkn_iter) + self.out.emit(first_tkn) emit_to(self.out, tkn_iter, "COMMA") label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon self.emit(") JUMP_TO_ERROR();\n") + return not always_true(first_tkn) + def error_no_pop( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - stack: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: next(tkn_iter) # LPAREN next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon self.out.emit_at("JUMP_TO_ERROR();", tkn) + return False def deopt_if( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - unused: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: self.out.emit_at("if ", tkn) - self.emit(next(tkn_iter)) + lparen = next(tkn_iter) + self.emit(lparen) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() emit_to(self.out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon self.emit(") {\n") self.emit("UOP_STAT_INC(uopcode, miss);\n") self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") + return not always_true(first_tkn) def exit_if( # type: ignore[override] self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - unused: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: self.out.emit_at("if ", tkn) - self.emit(next(tkn_iter)) + lparen = next(tkn_iter) + self.emit(lparen) + first_tkn = tkn_iter.peek() emit_to(self.out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon self.emit(") {\n") self.emit("UOP_STAT_INC(uopcode, miss);\n") self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") + return not always_true(first_tkn) def oparg( self, tkn: Token, - tkn_iter: Iterator[Token], + tkn_iter: TokenIterator, uop: Uop, - unused: Stack, + storage: Storage, inst: Instruction | None, - ) -> None: + ) -> bool: if not uop.name.endswith("_0") and not uop.name.endswith("_1"): self.emit(tkn) - return + return True amp = next(tkn_iter) if amp.text != "&": self.emit(tkn) self.emit(amp) - return + return True one = next(tkn_iter) assert one.text == "1" self.out.emit_at(uop.name[-1], tkn) + return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} try: emitter.out.start_line() @@ -160,19 +177,9 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: elif uop.properties.const_oparg >= 0: emitter.emit(f"oparg = {uop.properties.const_oparg};\n") emitter.emit(f"assert(oparg == CURRENT_OPARG());\n") - for var in reversed(uop.stack.inputs): - code, local = stack.pop(var) + code_list, storage = Storage.for_uop(stack, uop) + for code in code_list: emitter.emit(code) - if local.defined: - locals[local.name] = local - emitter.emit(stack.define_output_arrays(uop.stack.outputs)) - outputs: list[Local] = [] - for var in uop.stack.outputs: - if var.name in locals: - local = locals[var.name] - else: - local = Local.local(var) - outputs.append(local) for cache in uop.caches: if cache.name != "unused": if cache.size == 4: @@ -181,15 +188,10 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> None: type = f"uint{cache.size*16}_t " cast = f"uint{cache.size*16}_t" emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND();\n") - emitter.emit_tokens(uop, stack, None) - for output in outputs: - if output.name in uop.deferred_refs.values(): - # We've already spilled this when emitting tokens - output.cached = False - stack.push(output) + storage = emitter.emit_tokens(uop, storage, None) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) from None - + return storage.stack SKIPS = ("_EXTENDED_ARG",) @@ -226,7 +228,7 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - write_uop(uop, emitter, stack) + stack = write_uop(uop, emitter, stack) out.start_line() if not uop.properties.always_exits: stack.flush(out) diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 1c6a9edb39840d..bbb52f391f4b01 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -2,7 +2,6 @@ import dataclasses import enum -import sys import typing import _schema @@ -103,8 +102,8 @@ class HoleValue(enum.Enum): HoleValue.OPERAND_HI: "(instruction->operand >> 32)", HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", HoleValue.TARGET: "instruction->target", - HoleValue.JUMP_TARGET: "instruction_starts[instruction->jump_target]", - HoleValue.ERROR_TARGET: "instruction_starts[instruction->error_target]", + HoleValue.JUMP_TARGET: "state->instruction_starts[instruction->jump_target]", + HoleValue.ERROR_TARGET: "state->instruction_starts[instruction->error_target]", HoleValue.ZERO: "", } @@ -125,6 +124,7 @@ class Hole: symbol: str | None # ...plus this addend: addend: int + need_state: bool = False func: str = dataclasses.field(init=False) # Convenience method: replace = dataclasses.replace @@ -157,10 +157,12 @@ def as_c(self, where: str) -> str: if value: value += " + " value += f"(uintptr_t)&{self.symbol}" - if _signed(self.addend): + if _signed(self.addend) or not value: if value: value += " + " value += f"{_signed(self.addend):#x}" + if self.need_state: + return f"{self.func}({location}, {value}, state);" return f"{self.func}({location}, {value});" @@ -175,7 +177,6 @@ class Stencil: body: bytearray = dataclasses.field(default_factory=bytearray, init=False) holes: list[Hole] = dataclasses.field(default_factory=list, init=False) disassembly: list[str] = dataclasses.field(default_factory=list, init=False) - trampolines: dict[str, int] = dataclasses.field(default_factory=dict, init=False) def pad(self, alignment: int) -> None: """Pad the stencil to the given alignment.""" @@ -184,39 +185,6 @@ def pad(self, alignment: int) -> None: self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") self.body.extend([0] * padding) - def emit_aarch64_trampoline(self, hole: Hole, alignment: int) -> Hole: - """Even with the large code model, AArch64 Linux insists on 28-bit jumps.""" - assert hole.symbol is not None - reuse_trampoline = hole.symbol in self.trampolines - if reuse_trampoline: - # Re-use the base address of the previously created trampoline - base = self.trampolines[hole.symbol] - else: - self.pad(alignment) - base = len(self.body) - new_hole = hole.replace(addend=base, symbol=None, value=HoleValue.DATA) - - if reuse_trampoline: - return new_hole - - self.disassembly += [ - f"{base + 4 * 0:x}: 58000048 ldr x8, 8", - f"{base + 4 * 1:x}: d61f0100 br x8", - f"{base + 4 * 2:x}: 00000000", - f"{base + 4 * 2:016x}: R_AARCH64_ABS64 {hole.symbol}", - f"{base + 4 * 3:x}: 00000000", - ] - for code in [ - 0x58000048.to_bytes(4, sys.byteorder), - 0xD61F0100.to_bytes(4, sys.byteorder), - 0x00000000.to_bytes(4, sys.byteorder), - 0x00000000.to_bytes(4, sys.byteorder), - ]: - self.body.extend(code) - self.holes.append(hole.replace(offset=base + 8, kind="R_AARCH64_ABS64")) - self.trampolines[hole.symbol] = base - return new_hole - def remove_jump(self, *, alignment: int = 1) -> None: """Remove a zero-length continuation jump, if it exists.""" hole = max(self.holes, key=lambda hole: hole.offset) @@ -282,8 +250,14 @@ class StencilGroup: default_factory=dict, init=False ) _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) - - def process_relocations(self, *, alignment: int = 1) -> None: + _trampolines: set[int] = dataclasses.field(default_factory=set, init=False) + + def process_relocations( + self, + known_symbols: dict[str, int], + *, + alignment: int = 1, + ) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( @@ -291,9 +265,17 @@ def process_relocations(self, *, alignment: int = 1) -> None: in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} and hole.value is HoleValue.ZERO ): - new_hole = self.data.emit_aarch64_trampoline(hole, alignment) - self.code.holes.remove(hole) - self.code.holes.append(new_hole) + hole.func = "patch_aarch64_trampoline" + hole.need_state = True + assert hole.symbol is not None + if hole.symbol in known_symbols: + ordinal = known_symbols[hole.symbol] + else: + ordinal = len(known_symbols) + known_symbols[hole.symbol] = ordinal + self._trampolines.add(ordinal) + hole.addend = ordinal + hole.symbol = None self.code.remove_jump(alignment=alignment) self.code.pad(alignment) self.data.pad(8) @@ -348,9 +330,20 @@ def _emit_global_offset_table(self) -> None: ) self.data.body.extend([0] * 8) + def _get_trampoline_mask(self) -> str: + bitmask: int = 0 + trampoline_mask: list[str] = [] + for ordinal in self._trampolines: + bitmask |= 1 << ordinal + while bitmask: + word = bitmask & ((1 << 32) - 1) + trampoline_mask.append(f"{word:#04x}") + bitmask >>= 32 + return "{" + ", ".join(trampoline_mask) + "}" + def as_c(self, opname: str) -> str: """Dump this hole as a StencilGroup initializer.""" - return f"{{emit_{opname}, {len(self.code.body)}, {len(self.data.body)}}}" + return f"{{emit_{opname}, {len(self.code.body)}, {len(self.data.body)}, {self._get_trampoline_mask()}}}" def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index e37ee943999785..5eb316e782fda8 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -44,6 +44,7 @@ class _Target(typing.Generic[_S, _R]): stable: bool = False debug: bool = False verbose: bool = False + known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) def _compute_digest(self, out: pathlib.Path) -> str: hasher = hashlib.sha256() @@ -95,7 +96,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) - group.process_relocations(alignment=self.alignment) + group.process_relocations( + known_symbols=self.known_symbols, alignment=self.alignment + ) return group def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: @@ -231,7 +234,7 @@ def build( if comment: file.write(f"// {comment}\n") file.write("\n") - for line in _writer.dump(stencil_groups): + for line in _writer.dump(stencil_groups, self.known_symbols): file.write(f"{line}\n") try: jit_stencils_new.replace(jit_stencils) @@ -524,7 +527,12 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: args = ["-fms-runtime-lib=dll"] target = _COFF(host, alignment=8, args=args) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): - args = ["-fpic"] + args = [ + "-fpic", + # On aarch64 Linux, intrinsics were being emitted and this flag + # was required to disable them. + "-mno-outline-atomics", + ] target = _ELF(host, alignment=8, args=args) elif re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 9d11094f85c7ff..7b99d10310a645 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -2,17 +2,24 @@ import itertools import typing +import math import _stencils -def _dump_footer(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: +def _dump_footer( + groups: dict[str, _stencils.StencilGroup], symbols: dict[str, int] +) -> typing.Iterator[str]: + symbol_mask_size = max(math.ceil(len(symbols) / 32), 1) + yield f'static_assert(SYMBOL_MASK_WORDS >= {symbol_mask_size}, "SYMBOL_MASK_WORDS too small");' + yield "" yield "typedef struct {" yield " void (*emit)(" yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," - yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[]);" + yield " const _PyUOpInstruction *instruction, jit_state *state);" yield " size_t code_size;" yield " size_t data_size;" + yield " symbol_mask trampoline_mask;" yield "} StencilGroup;" yield "" yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" @@ -23,13 +30,18 @@ def _dump_footer(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[s continue yield f" [{opname}] = {group.as_c(opname)}," yield "};" + yield "" + yield f"static const void * const symbols_map[{max(len(symbols), 1)}] = {{" + for symbol, ordinal in symbols.items(): + yield f" [{ordinal}] = &{symbol}," + yield "};" def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: yield "void" yield f"emit_{opname}(" yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," - yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[])" + yield " const _PyUOpInstruction *instruction, jit_state *state)" yield "{" for part, stencil in [("code", group.code), ("data", group.data)]: for line in stencil.disassembly: @@ -58,8 +70,10 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator yield "" -def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: +def dump( + groups: dict[str, _stencils.StencilGroup], symbols: dict[str, int] +) -> typing.Iterator[str]: """Yield a JIT compiler line-by-line as a C header file.""" for opname, group in sorted(groups.items()): yield from _dump_stencil(opname, group) - yield from _dump_footer(groups) + yield from _dump_footer(groups, symbols) diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt index dbb364673b5c1a..e379e39def0eaf 100644 --- a/Tools/jit/ignore-tests-emulated-linux.txt +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -1,4 +1,5 @@ test_multiprocessing_fork +test_strftime_y2k test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 0014204e89d1bb..49f681d3e11d2e 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -123,7 +123,7 @@ Feel free to post at <a href="https://discuss.python.org/c/users/7">discus You must restart your computer to complete the rollback of the software. &Restart Unable to install [WixBundleName] due to an existing install. Use Programs and Features to modify, repair or remove [WixBundleName]. - At least Windows 8.1 or Windows Server 2012 are required to install [WixBundleName] + At least Windows 10 or Windows Server 2016 are required to install [WixBundleName] Visit <a href="https://www.python.org/downloads/">python.org</a> to download an earlier version of Python. Disable path length limit diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index 094ddba4f1ad8f..1e0df5084ff079 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -3086,11 +3086,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; if (IsWindowsServer()) { - if (IsWindowsVersionOrGreater(6, 2, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2012 or later"); + if (IsWindowsVersionOrGreater(10, 0, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2016 or later"); return; + } else if (IsWindowsVersionOrGreater(6, 2, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2012"); } else if (IsWindowsVersionOrGreater(6, 1, 1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected Windows Server 2008 R2"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 1, 0)) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 0, 0)) { @@ -3098,14 +3100,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2003 or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2012 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2016 or later is required to continue installation"); } else { if (IsWindows10OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 10 or later"); return; } else if (IsWindows8Point1OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 8.1"); - return; + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8.1"); } else if (IsWindows8OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8"); } else if (IsWindows7OrGreater()) { @@ -3115,7 +3116,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows XP or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 8.1 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 10 or later is required to continue installation"); } LocGetString(_wixLoc, L"#(loc.FailureOldOS)", &pLocString); diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index e25219a6caf9d4..4a13d368d7c744 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -51,14 +51,6 @@ "test_pdb.msi", "tools.msi", "ucrt.msi", - "Windows6.0-KB2999226-x64.msu", - "Windows6.0-KB2999226-x86.msu", - "Windows6.1-KB2999226-x64.msu", - "Windows6.1-KB2999226-x86.msu", - "Windows8.1-KB2999226-x64.msu", - "Windows8.1-KB2999226-x86.msu", - "Windows8-RT-KB2999226-x64.msu", - "Windows8-RT-KB2999226-x86.msu", ] PATHS = [ "python-{}.exe".format(m.group(0)), diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 547c55dce130f7..a4951d05e80ebd 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -3,7 +3,7 @@ import re from dataclasses import dataclass, field from enum import Enum -from typing import IO, Any, Dict, List, Optional, Set, Text, Tuple +from typing import IO, Any, Callable, Dict, List, Optional, Set, Text, Tuple from pegen import grammar from pegen.grammar import ( @@ -130,7 +130,7 @@ def __init__( self.gen = parser_generator self.exact_tokens = exact_tokens self.non_exact_tokens = non_exact_tokens - self.cache: Dict[Any, FunctionCall] = {} + self.cache: Dict[str, str] = {} self.cleanup_statements: List[str] = [] def keyword_helper(self, keyword: str) -> FunctionCall: @@ -206,21 +206,6 @@ def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: comment=f"token='{val}'", ) - def visit_Rhs(self, node: Rhs) -> FunctionCall: - if node in self.cache: - return self.cache[node] - if node.can_be_inlined: - self.cache[node] = self.generate_call(node.alts[0].items[0]) - else: - name = self.gen.artificial_rule_from_rhs(node) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - comment=f"{node}", - ) - return self.cache[node] - def visit_NamedItem(self, node: NamedItem) -> FunctionCall: call = self.generate_call(node.item) if node.name: @@ -302,44 +287,62 @@ def visit_Opt(self, node: Opt) -> FunctionCall: comment=f"{node}", ) - def visit_Repeat0(self, node: Repeat0) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, False) - self.cache[node] = FunctionCall( + def _generate_artificial_rule_call( + self, + node: Any, + prefix: str, + rule_generation_func: Callable[[], str], + return_type: Optional[str] = None, + ) -> FunctionCall: + node_str = f"{node}" + key = f"{prefix}_{node_str}" + if key in self.cache: + name = self.cache[key] + else: + name = rule_generation_func() + self.cache[key] = name + + return FunctionCall( assigned_variable=f"{name}_var", function=f"{name}_rule", arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return_type=return_type, + comment=node_str, + ) + + def visit_Rhs(self, node: Rhs) -> FunctionCall: + if node.can_be_inlined: + return self.generate_call(node.alts[0].items[0]) + + return self._generate_artificial_rule_call( + node, + "rhs", + lambda: self.gen.artificial_rule_from_rhs(node), + ) + + def visit_Repeat0(self, node: Repeat0) -> FunctionCall: + return self._generate_artificial_rule_call( + node, + "repeat0", + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), + "asdl_seq *", ) - return self.cache[node] def visit_Repeat1(self, node: Repeat1) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, True) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return self._generate_artificial_rule_call( + node, + "repeat1", + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), + "asdl_seq *", ) - return self.cache[node] def visit_Gather(self, node: Gather) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_gather(node) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return self._generate_artificial_rule_call( + node, + "gather", + lambda: self.gen.artificial_rule_from_gather(node), + "asdl_seq *", ) - return self.cache[node] def visit_Group(self, node: Group) -> FunctionCall: return self.generate_call(node.rhs) diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 8cca7b6c39a5cc..b42b12c8aa0dee 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -184,8 +184,6 @@ def artificial_rule_from_repeat(self, node: Plain, is_repeat1: bool) -> str: return name def artificial_rule_from_gather(self, node: Gather) -> str: - self.counter += 1 - name = f"_gather_{self.counter}" self.counter += 1 extra_function_name = f"_loop0_{self.counter}" extra_function_alt = Alt( @@ -197,6 +195,8 @@ def artificial_rule_from_gather(self, node: Gather) -> str: None, Rhs([extra_function_alt]), ) + self.counter += 1 + name = f"_gather_{self.counter}" alt = Alt( [NamedItem("elem", node.node), NamedItem("seq", NameLeaf(extra_function_name))], ) diff --git a/Tools/wasm/Setup.local.example b/Tools/wasm/Setup.local.example deleted file mode 100644 index 7b2fb13f6ceef2..00000000000000 --- a/Tools/wasm/Setup.local.example +++ /dev/null @@ -1,13 +0,0 @@ -# Module/Setup.local with reduced stdlib -*disabled* -_asyncio -_bz2 -_decimal -_pickle -pyexpat _elementtree -_sha3 _blake2 -_zoneinfo -xxsubtype - -# cjk codecs -#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw diff --git a/Tools/wasm/build_wasi.sh b/Tools/wasm/build_wasi.sh deleted file mode 100755 index 436306222ce1d0..00000000000000 --- a/Tools/wasm/build_wasi.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/bash - -set -e -x - -# Quick check to avoid running configure just to fail in the end. -if [ -f Programs/python.o ]; then - echo "Can't do an out-of-tree build w/ an in-place build pre-existing (i.e., found Programs/python.o)" >&2 - exit 1 -fi - -if [ ! -f Modules/Setup.local ]; then - touch Modules/Setup.local -fi - -# TODO: check if `make` and `pkgconfig` are installed -# TODO: detect if wasmtime is installed - -# Create the "build" Python. -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -s -j 4 all -export PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` -popd - -# Create the "host"/WASI Python. -export CONFIG_SITE="$(pwd)/Tools/wasm/config.site-wasm32-wasi" -export HOSTRUNNER="wasmtime run --mapdir /::$(pwd) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/builddir/wasi/python.wasm --" - -mkdir -p builddir/wasi -pushd builddir/wasi -../../Tools/wasm/wasi-env \ - ../../configure \ - -C \ - --host=wasm32-unknown-wasi \ - --build=$(../../config.guess) \ - --with-build-python=../build/python -make -s -j 4 all -# Create a helper script for executing the host/WASI Python. -printf "#!/bin/sh\nexec $HOSTRUNNER \"\$@\"\n" > run_wasi.sh -chmod 755 run_wasi.sh -./run_wasi.sh --version -popd - diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 95eda863cb62c6..4c5078a1f675e2 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,6 +1,8 @@ #!/bin/sh set -e +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . + # function usage() { echo "wasi-env - Run command with WASI-SDK" diff --git a/configure b/configure index 0cc73e4e66552d..be119f108a060b 100755 --- a/configure +++ b/configure @@ -11092,6 +11092,12 @@ if test "x$ac_cv_header_linux_memfd_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_MEMFD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/netfilter_ipv4.h" "ac_cv_header_linux_netfilter_ipv4_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_netfilter_ipv4_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_NETFILTER_IPV4_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/random.h" "ac_cv_header_linux_random_h" "$ac_includes_default" if test "x$ac_cv_header_linux_random_h" = xyes @@ -28993,10 +28999,6 @@ printf %s "checking whether libatomic is needed by ... " >&6; } if test ${ac_cv_libatomic_needed+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes -then : - ac_cv_libatomic_needed=no else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -29038,16 +29040,14 @@ int main() } _ACEOF -if ac_fn_c_try_run "$LINENO" +if ac_fn_c_try_link "$LINENO" then : ac_cv_libatomic_needed=no else $as_nop ac_cv_libatomic_needed=yes fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 printf "%s\n" "$ac_cv_libatomic_needed" >&6; } diff --git a/configure.ac b/configure.ac index 1864e94ace9243..582851695e400f 100644 --- a/configure.ac +++ b/configure.ac @@ -3013,7 +3013,7 @@ AC_DEFINE([STDC_HEADERS], [1], AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ - linux/random.h linux/soundcard.h \ + linux/netfilter_ipv4.h linux/random.h linux/soundcard.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ @@ -7497,7 +7497,7 @@ CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" AC_CACHE_CHECK([whether libatomic is needed by ], [ac_cv_libatomic_needed], -[AC_RUN_IFELSE([AC_LANG_SOURCE([[ +[AC_LINK_IFELSE([AC_LANG_SOURCE([[ // pyatomic.h needs uint64_t and Py_ssize_t types #include // int64_t, intptr_t #ifdef HAVE_SYS_TYPES_H @@ -7534,9 +7534,8 @@ int main() return 0; // all good } ]])], - [ac_cv_libatomic_needed=no], dnl build succeeded - [ac_cv_libatomic_needed=yes], dnl build failed - [ac_cv_libatomic_needed=no]) dnl cross compilation + [ac_cv_libatomic_needed=no], dnl build and link succeeded + [ac_cv_libatomic_needed=yes]) dnl build and link failed ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], diff --git a/pyconfig.h.in b/pyconfig.h.in index 7f02603e26f5d0..1947d8ee14f83e 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -739,6 +739,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_MEMFD_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_NETFILTER_IPV4_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_NETLINK_H