diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0e0fb8c1152f4..35d9c64a8c5c15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.0 + rev: v0.1.2 hooks: - id: ruff name: Run Ruff on Lib/test/ diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index aed4ae44c76eea..7198d6bc056eb4 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -108,6 +108,8 @@ This is a pointer to a function with the following signature: Doing so will allow callables such as bound methods to make their onward calls (which include a prepended *self* argument) very efficiently. + .. versionadded:: 3.8 + To call an object that implements vectorcall, use a :ref:`call API ` function as with any other callable. :c:func:`PyObject_Vectorcall` will usually be most efficient. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 5ab9f1cab23ef8..5fa37963e07eff 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -971,8 +971,8 @@ These are the UTF-8 codec APIs: returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, ``NULL`` is returned with an exception set and no - *size* is stored. + On error, set an exception, set *size* to ``-1`` (if it's not NULL) and + return ``NULL``. This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not @@ -992,11 +992,19 @@ These are the UTF-8 codec APIs: As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. + Raise an exception if the *unicode* string contains embedded null + characters. To accept embedded null characters and truncate on purpose + at the first null byte, ``PyUnicode_AsUTF8AndSize(unicode, NULL)`` can be + used instead. + .. versionadded:: 3.3 .. versionchanged:: 3.7 The return type is now ``const char *`` rather of ``char *``. + .. versionchanged:: 3.13 + Raise an exception if the string contains embedded null characters. + UTF-32 Codecs """"""""""""" diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 811b1bd84d2417..52d6d967d66327 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -726,6 +726,7 @@ function,PyUnicode_AsUCS4,3.7,, function,PyUnicode_AsUCS4Copy,3.7,, function,PyUnicode_AsUTF16String,3.2,, function,PyUnicode_AsUTF32String,3.2,, +function,PyUnicode_AsUTF8,3.13,, function,PyUnicode_AsUTF8AndSize,3.10,, function,PyUnicode_AsUTF8String,3.2,, function,PyUnicode_AsUnicodeEscapeString,3.2,, diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index c19c48301f5848..5e2f9a9d1837fe 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -245,6 +245,9 @@ You can omit either *m* or *n*; in that case, a reasonable value is assumed for the missing value. Omitting *m* is interpreted as a lower limit of 0, while omitting *n* results in an upper bound of infinity. +The simplest case ``{m}`` matches the preceding item exactly *m* times. +For example, ``a/{2}b`` will only match ``'a//b'``. + Readers of a reductionist bent may notice that the three other quantifiers can all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}`` is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 361e7bb9c8f2fa..75c459c0cb601f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -243,9 +243,9 @@ Scheduling callbacks See the :ref:`concurrency and multithreading ` section of the documentation. -.. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. + .. versionchanged:: 3.7 + The *context* keyword-only parameter was added. See :pep:`567` + for more details. .. _asyncio-pass-keywords: @@ -661,6 +661,8 @@ Opening network connections Creating network servers ^^^^^^^^^^^^^^^^^^^^^^^^ +.. _loop_create_server: + .. coroutinemethod:: loop.create_server(protocol_factory, \ host=None, port=None, *, \ family=socket.AF_UNSPEC, \ @@ -1191,6 +1193,8 @@ Working with pipes Unix signals ^^^^^^^^^^^^ +.. _loop_add_signal_handler: + .. method:: loop.add_signal_handler(signum, callback, *args) Set *callback* as the handler for the *signum* signal. @@ -1391,6 +1395,14 @@ Enabling debug mode The new :ref:`Python Development Mode ` can now also be used to enable the debug mode. +.. attribute:: loop.slow_callback_duration + + This attribute can be used to set the + minimum execution duration in seconds that is considered "slow". + When debug mode is enabled, "slow" callbacks are logged. + + Default value is 100 milliseconds. + .. seealso:: The :ref:`debug mode of asyncio `. @@ -1411,6 +1423,8 @@ async/await code consider using the high-level :ref:`Subprocess Support on Windows ` for details. +.. _loop_subprocess_exec: + .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 88fdf41c7b2a3e..797065c8ccf894 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -592,7 +592,7 @@ Shielding From Cancellation is equivalent to:: - res = await shield(something()) + res = await something() *except* that if the coroutine containing it is cancelled, the Task running in ``something()`` is not cancelled. From the point diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index c75ab47404c1e4..5f33c6813e74c0 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -46,9 +46,9 @@ Additionally, there are **low-level** APIs for *library and framework developers* to: * create and manage :ref:`event loops `, which - provide asynchronous APIs for :meth:`networking `, - running :meth:`subprocesses `, - handling :meth:`OS signals `, etc; + provide asynchronous APIs for :ref:`networking `, + running :ref:`subprocesses `, + handling :ref:`OS signals `, etc; * implement efficient protocols using :ref:`transports `; diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ec4aeaa04395ac..6a95a4a6e8d183 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -91,7 +91,7 @@ The :mod:`bz2` module contains: and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - :class:`BZ2File` also provides the following method: + :class:`BZ2File` also provides the following methods: .. method:: peek([n]) @@ -106,14 +106,52 @@ The :mod:`bz2` module contains: .. versionadded:: 3.3 + .. method:: fileno() + + Return the file descriptor for the underlying file. + + .. versionadded:: 3.3 + + .. method:: readable() + + Return whether the file was opened for reading. + + .. versionadded:: 3.3 + + .. method:: seekable() + + Return whether the file supports seeking. + + .. versionadded:: 3.3 + + .. method:: writable() + + Return whether the file was opened for writing. + + .. versionadded:: 3.3 + + .. method:: read1(size=-1) + + Read up to *size* uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns ``b''`` if the file is at EOF. + + .. versionadded:: 3.3 + + .. method:: readinto(b) + + Read bytes into *b*. + + Returns the number of bytes read (0 for EOF). + + .. versionadded:: 3.3 + .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. - .. versionchanged:: 3.3 - The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`, - :meth:`read1` and :meth:`readinto` methods were added. - .. versionchanged:: 3.3 Support was added for *filename* being a :term:`file object` instead of an actual filename. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index d5f46ed58e8d24..2d5ae361c3f1e3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -165,7 +165,7 @@ Data Types to subclass *EnumType* -- see :ref:`Subclassing EnumType ` for details. - *EnumType* is responsible for setting the correct :meth:`!__repr__`, + ``EnumType`` is responsible for setting the correct :meth:`!__repr__`, :meth:`!__str__`, :meth:`!__format__`, and :meth:`!__reduce__` methods on the final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. @@ -198,11 +198,12 @@ Data Types >>> some_var = Color.RED >>> some_var in Color True + >>> Color.RED.value in Color + True - .. note:: + .. versionchanged:: 3.12 - In Python 3.12 it will be possible to check for member values and not - just members; until then, a ``TypeError`` will be raised if a + Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. .. method:: EnumType.__dir__(cls) @@ -405,7 +406,7 @@ Data Types .. class:: IntEnum - *IntEnum* is the same as *Enum*, but its members are also integers and can be + *IntEnum* is the same as :class:`Enum`, but its members are also integers and can be used anywhere that an integer can be used. If any integer operation is performed with an *IntEnum* member, the resulting value loses its enumeration status. @@ -436,7 +437,7 @@ Data Types .. class:: StrEnum - *StrEnum* is the same as *Enum*, but its members are also strings and can be used + ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used in most of the same places that a string can be used. The result of any string operation performed on or with a *StrEnum* member is not part of the enumeration. @@ -575,7 +576,7 @@ Data Types .. class:: IntFlag - *IntFlag* is the same as *Flag*, but its members are also integers and can be + ``IntFlag`` is the same as :class:`Flag`, but its members are also integers and can be used anywhere that an integer can be used. >>> from enum import IntFlag, auto @@ -595,12 +596,12 @@ Data Types >>> Color.RED + 2 3 - If a *Flag* operation is performed with an *IntFlag* member and: + If a :class:`Flag` operation is performed with an *IntFlag* member and: * the result is a valid *IntFlag*: an *IntFlag* is returned - * the result is not a valid *IntFlag*: the result depends on the *FlagBoundary* setting + * the result is not a valid *IntFlag*: the result depends on the :class:`FlagBoundary` setting - The *repr()* of unnamed zero-valued flags has changed. It is now: + The :func:`repr()` of unnamed zero-valued flags has changed. It is now: >>> Color(0) @@ -696,7 +697,7 @@ Data Types .. class:: FlagBoundary - *FlagBoundary* controls how out-of-range values are handled in *Flag* and its + ``FlagBoundary`` controls how out-of-range values are handled in :class:`Flag` and its subclasses. .. attribute:: STRICT @@ -719,7 +720,7 @@ Data Types .. attribute:: CONFORM - Out-of-range values have invalid values removed, leaving a valid *Flag* + Out-of-range values have invalid values removed, leaving a valid :class:`Flag` value:: >>> from enum import Flag, CONFORM, auto @@ -733,7 +734,7 @@ Data Types .. attribute:: EJECT - Out-of-range values lose their *Flag* membership and revert to :class:`int`. + Out-of-range values lose their :class:`Flag` membership and revert to :class:`int`. >>> from enum import Flag, EJECT, auto >>> class EjectFlag(Flag, boundary=EJECT): @@ -746,7 +747,7 @@ Data Types .. attribute:: KEEP - Out-of-range values are kept, and the *Flag* membership is kept. + Out-of-range values are kept, and the :class:`Flag` membership is kept. This is the default for :class:`IntFlag`:: >>> from enum import Flag, KEEP, auto @@ -808,10 +809,10 @@ Utilities and Decorators .. class:: auto *auto* can be used in place of a value. If used, the *Enum* machinery will - call an *Enum*'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For *Enum* and *IntEnum* that appropriate value will be the last value plus - one; for *Flag* and *IntFlag* it will be the first power-of-two greater - than the highest value; for *StrEnum* it will be the lower-cased version of + call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus + one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3efd3be5909254..b463c0b6d0e402 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -640,6 +640,9 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. + If the passed object has a ``__signature__`` attribute, this function + returns it without further computations. + For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using @@ -763,6 +766,8 @@ function. sig = MySignature.from_callable(min) assert isinstance(sig, MySignature) + Its behavior is otherwise identical to that of :func:`signature`. + .. versionadded:: 3.5 .. versionadded:: 3.10 diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index afd5677deac3f8..865969e7d121b3 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -303,7 +303,7 @@ The :mod:`locale` module defines the following exception and functions: *language code* and *encoding* may be ``None`` if their values cannot be determined. - .. deprecated-removed:: 3.11 3.13 + .. deprecated-removed:: 3.11 3.15 .. function:: getlocale(category=LC_CTYPE) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 69afadff1f5f42..4ca7a64451d4c7 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -19,7 +19,7 @@ the current file position, and :meth:`seek` through the file to different positi A memory-mapped file is created by the :class:`~mmap.mmap` constructor, which is different on Unix and on Windows. In either case you must provide a file descriptor for a file opened for update. If you wish to map an existing Python -file object, use its :meth:`fileno` method to obtain the correct value for the +file object, use its :meth:`~io.IOBase.fileno` method to obtain the correct value for the *fileno* parameter. Otherwise, you can open the file using the :func:`os.open` function, which returns a file descriptor directly (the file still needs to be closed when done). diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 231038e6678dbc..789a84b02d59d2 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2582,7 +2582,7 @@ multiple connections at the same time. **Windows**: An item in *object_list* must either be an integer handle which is waitable (according to the definition used by the documentation of the Win32 function ``WaitForMultipleObjects()``) - or it can be an object with a :meth:`fileno` method which returns a + or it can be an object with a :meth:`~io.IOBase.fileno` method which returns a socket handle or pipe handle. (Note that pipe handles and socket handles are **not** waitable handles.) diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index dd50bac37e49b8..76cbf91412f763 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -21,7 +21,7 @@ It defines a :class:`BaseSelector` abstract base class, along with several concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), that can be used to wait for I/O readiness notification on multiple file objects. In the following, "file object" refers to any object with a -:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. +:meth:`~io.IOBase.fileno` method, or a raw file descriptor. See :term:`file object`. :class:`DefaultSelector` is an alias to the most efficient implementation available on the current platform: this should be the default choice for most diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c3cf48316b3248..14f11b40cb9b00 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -858,7 +858,7 @@ The following functions all create :ref:`socket objects `. .. function:: fromfd(fd, family, type, proto=0) Duplicate the file descriptor *fd* (an integer as returned by a file object's - :meth:`fileno` method) and build a socket object from the result. Address + :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address family, socket type and protocol number are as for the :func:`.socket` function above. The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index fc87aa94d3363b..a5b3474f4bd39a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1154,6 +1154,10 @@ Connection objects f.write('%s\n' % line) con.close() + .. seealso:: + + :ref:`sqlite3-howto-encoding` + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) @@ -1220,6 +1224,10 @@ Connection objects .. versionadded:: 3.7 + .. seealso:: + + :ref:`sqlite3-howto-encoding` + .. method:: getlimit(category, /) Get a connection runtime limit. @@ -1441,39 +1449,8 @@ Connection objects and returns a text representation of it. The callable is invoked for SQLite values with the ``TEXT`` data type. By default, this attribute is set to :class:`str`. - If you want to return ``bytes`` instead, set *text_factory* to ``bytes``. - Example: - - .. testcode:: - - con = sqlite3.connect(":memory:") - cur = con.cursor() - - AUSTRIA = "Österreich" - - # by default, rows are returned as str - cur.execute("SELECT ?", (AUSTRIA,)) - row = cur.fetchone() - assert row[0] == AUSTRIA - - # but we can make sqlite3 always return bytestrings ... - con.text_factory = bytes - cur.execute("SELECT ?", (AUSTRIA,)) - row = cur.fetchone() - assert type(row[0]) is bytes - # the bytestrings will be encoded in UTF-8, unless you stored garbage in the - # database ... - assert row[0] == AUSTRIA.encode("utf-8") - - # we can also implement a custom text_factory ... - # here we implement one that appends "foo" to all strings - con.text_factory = lambda x: x.decode("utf-8") + "foo" - cur.execute("SELECT ?", ("bar",)) - row = cur.fetchone() - assert row[0] == "barfoo" - - con.close() + See :ref:`sqlite3-howto-encoding` for more details. .. attribute:: total_changes @@ -1632,7 +1609,6 @@ Cursor objects COMMIT; """) - .. method:: fetchone() If :attr:`~Cursor.row_factory` is ``None``, @@ -2611,6 +2587,47 @@ With some adjustments, the above recipe can be adapted to use a instead of a :class:`~collections.namedtuple`. +.. _sqlite3-howto-encoding: + +How to handle non-UTF-8 text encodings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, :mod:`!sqlite3` uses :class:`str` to adapt SQLite values +with the ``TEXT`` data type. +This works well for UTF-8 encoded text, but it might fail for other encodings +and invalid UTF-8. +You can use a custom :attr:`~Connection.text_factory` to handle such cases. + +Because of SQLite's `flexible typing`_, it is not uncommon to encounter table +columns with the ``TEXT`` data type containing non-UTF-8 encodings, +or even arbitrary data. +To demonstrate, let's assume we have a database with ISO-8859-2 (Latin-2) +encoded text, for example a table of Czech-English dictionary entries. +Assuming we now have a :class:`Connection` instance :py:data:`!con` +connected to this database, +we can decode the Latin-2 encoded text using this :attr:`~Connection.text_factory`: + +.. testcode:: + + con.text_factory = lambda data: str(data, encoding="latin2") + +For invalid UTF-8 or arbitrary data in stored in ``TEXT`` table columns, +you can use the following technique, borrowed from the :ref:`unicode-howto`: + +.. testcode:: + + con.text_factory = lambda data: str(data, errors="surrogateescape") + +.. note:: + + The :mod:`!sqlite3` module API does not support strings + containing surrogates. + +.. seealso:: + + :ref:`unicode-howto` + + .. _sqlite3-explanation: Explanation diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 1024f66f3264ba..f2fe3d76248c2c 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -44,21 +44,21 @@ Identifiers are integers in the range 0 to 5 inclusive. Registering and using tools ''''''''''''''''''''''''''' -.. function:: use_tool_id(id: int, name: str) -> None +.. function:: use_tool_id(tool_id: int, name: str, /) -> None - Must be called before *id* can be used. - *id* must be in the range 0 to 5 inclusive. - Raises a :exc:`ValueError` if *id* is in use. + Must be called before *tool_id* can be used. + *tool_id* must be in the range 0 to 5 inclusive. + Raises a :exc:`ValueError` if *tool_id* is in use. -.. function:: free_tool_id(id: int) -> None +.. function:: free_tool_id(tool_id: int, /) -> None - Should be called once a tool no longer requires *id*. + Should be called once a tool no longer requires *tool_id*. -.. function:: get_tool(id: int) -> str | None +.. function:: get_tool(tool_id: int, /) -> str | None - Returns the name of the tool if *id* is in use, + Returns the name of the tool if *tool_id* is in use, otherwise it returns ``None``. - *id* must be in the range 0 to 5 inclusive. + *tool_id* must be in the range 0 to 5 inclusive. All IDs are treated the same by the VM with regard to events, but the following IDs are pre-defined to make co-operation of tools easier:: @@ -237,11 +237,11 @@ Setting events globally Events can be controlled globally by modifying the set of events being monitored. -.. function:: get_events(tool_id: int) -> int +.. function:: get_events(tool_id: int, /) -> int Returns the ``int`` representing all the active events. -.. function:: set_events(tool_id: int, event_set: int) +.. function:: set_events(tool_id: int, event_set: int, /) -> None Activates all events which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not in use. @@ -253,11 +253,11 @@ Per code object events Events can also be controlled on a per code object basis. -.. function:: get_local_events(tool_id: int, code: CodeType) -> int +.. function:: get_local_events(tool_id: int, code: CodeType, /) -> int Returns all the local events for *code* -.. function:: set_local_events(tool_id: int, code: CodeType, event_set: int) +.. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None Activates all the local events for *code* which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not in use. @@ -284,6 +284,11 @@ performance monitoring. For example, a program can be run under a debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +.. function:: restart_events() -> None + + Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` + for all tools. + .. _callbacks: @@ -292,7 +297,7 @@ Registering callback functions To register a callable for events call -.. function:: register_callback(tool_id: int, event: int, func: Callable | None) -> Callable | None +.. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None Registers the callable *func* for the *event* with the given *tool_id* diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 8eeeefa0133d12..5ef6f83030958f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1780,7 +1780,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:!buffer` attribute. + do not support the :attr:`!buffer` attribute. .. data:: __stdin__ diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 42314648fec2bd..b2baa54d9522df 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -147,7 +147,7 @@ The module defines the following user-callable items: This class operates exactly as :func:`TemporaryFile` does, except that data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the + until the file's :func:`~io.IOBase.fileno` method is called, at which point the contents are written to disk and operation proceeds as with :func:`TemporaryFile`. diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index fb1ff567d49e5c..03806178e9d3fb 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -43,10 +43,20 @@ The module defines the following functions: Set the tty attributes for file descriptor *fd* from the *attributes*, which is a list like the one returned by :func:`tcgetattr`. The *when* argument - determines when the attributes are changed: :const:`TCSANOW` to change - immediately, :const:`TCSADRAIN` to change after transmitting all queued output, - or :const:`TCSAFLUSH` to change after transmitting all queued output and - discarding all queued input. + determines when the attributes are changed: + + .. data:: TCSANOW + + Change attributes immediately. + + .. data:: TCSADRAIN + + Change attributes after transmitting all queued output. + + .. data:: TCSAFLUSH + + Change attributes after transmitting all queued output and + discarding all queued input. .. function:: tcsendbreak(fd, duration) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 6be77260faa7ea..6e962881f29c49 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -508,34 +508,6 @@ The :mod:`test.support` module defines the following functions: Define match patterns on test filenames and test method names for filtering tests. -.. function:: run_unittest(*classes) - - Execute :class:`unittest.TestCase` subclasses passed to the function. The - function scans the classes for methods starting with the prefix ``test_`` - and executes the tests individually. - - It is also legal to pass strings as parameters; these should be keys in - ``sys.modules``. Each associated module will be scanned by - ``unittest.TestLoader.loadTestsFromModule()``. This is usually seen in the - following :func:`test_main` function:: - - def test_main(): - support.run_unittest(__name__) - - This will run all tests defined in the named module. - - -.. function:: run_doctest(module, verbosity=None, optionflags=0) - - Run :func:`doctest.testmod` on the given *module*. Return - ``(failure_count, test_count)``. - - If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity - set to :data:`verbose`. Otherwise, it is run with verbosity set to - ``None``. *optionflags* is passed as ``optionflags`` to - :func:`doctest.testmod`. - - .. function:: get_pagesize() Get size of a page in bytes. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 8f691932201225..bdff2bb776d844 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -304,7 +304,7 @@ a callable with any arbitrary parameter list would be acceptable: x = concat # Also OK ``Callable`` cannot express complex signatures such as functions that take a -variadic number of arguments, :func:`overloaded functions `, or +variadic number of arguments, :ref:`overloaded functions `, or functions that have keyword-only parameters. However, these signatures can be expressed by defining a :class:`Protocol` class with a :meth:`~object.__call__` method: @@ -526,7 +526,7 @@ A user-defined class can be defined as a generic class. self.logger.info('%s: %s', self.name, message) This syntax indicates that the class ``LoggedVar`` is parameterised around a -single :class:`type variable ` ``T`` . This also makes ``T`` valid as +single :ref:`type variable ` ``T`` . This also makes ``T`` valid as a type within the class body. Generic classes implicitly inherit from :class:`Generic`. For compatibility @@ -1493,7 +1493,7 @@ These can be used as types in annotations. They all support subscription using Typing operator to conceptually mark an object as having been unpacked. For example, using the unpack operator ``*`` on a - :class:`type variable tuple ` is equivalent to using ``Unpack`` + :ref:`type variable tuple ` is equivalent to using ``Unpack`` to mark the type variable tuple as having been unpacked:: Ts = TypeVarTuple('Ts') @@ -1584,6 +1584,8 @@ without the dedicated syntax, as documented below. ... # Etc. +.. _typevar: + .. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) Type variable. @@ -1728,9 +1730,11 @@ without the dedicated syntax, as documented below. :ref:`type parameter ` syntax introduced by :pep:`695`. The ``infer_variance`` parameter was added. +.. _typevartuple: + .. class:: TypeVarTuple(name) - Type variable tuple. A specialized form of :class:`type variable ` + Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. Type variable tuples can be declared in :ref:`type parameter lists ` @@ -1848,7 +1852,7 @@ without the dedicated syntax, as documented below. .. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) Parameter specification variable. A specialized version of - :class:`type variables `. + :ref:`type variables `. In :ref:`type parameter lists `, parameter specifications can be declared with two asterisks (``**``):: @@ -2772,6 +2776,8 @@ Functions and decorators .. versionadded:: 3.11 +.. _overload: + .. decorator:: overload Decorator for creating overloaded functions and methods. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 54c93008503c02..c9b32632c6b309 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -154,6 +154,7 @@ elements, call :meth:`XMLPullParser.read_events`. Here is an example:: ... print(elem.tag, 'text=', elem.text) ... end + mytag text= sometext more text The obvious use case is applications that operate in a non-blocking fashion where the XML data is being received from a socket or read incrementally from diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f44f9e65b9b530..4433b4ddfbbf72 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -30,7 +30,6 @@ Doc/howto/logging.rst Doc/howto/urllib2.rst Doc/library/abc.rst Doc/library/ast.rst -Doc/library/asyncio-dev.rst Doc/library/asyncio-eventloop.rst Doc/library/asyncio-extending.rst Doc/library/asyncio-policy.rst @@ -39,7 +38,6 @@ Doc/library/asyncio-subprocess.rst Doc/library/asyncio-task.rst Doc/library/bdb.rst Doc/library/bisect.rst -Doc/library/bz2.rst Doc/library/calendar.rst Doc/library/cmd.rst Doc/library/collections.abc.rst @@ -101,7 +99,6 @@ Doc/library/reprlib.rst Doc/library/resource.rst Doc/library/rlcompleter.rst Doc/library/select.rst -Doc/library/selectors.rst Doc/library/shelve.rst Doc/library/signal.rst Doc/library/smtplib.rst @@ -119,7 +116,6 @@ Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst Doc/library/traceback.rst -Doc/library/tty.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst @@ -141,7 +137,6 @@ Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst Doc/tutorial/datastructures.rst -Doc/tutorial/introduction.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 172611dd0cfb41..4536ab9486d39c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -428,7 +428,7 @@ type, i.e. it is possible to change their content:: [1, 8, 27, 64, 125] You can also add new items at the end of the list, by using -the :meth:`~list.append` *method* (we will see more about methods later):: +the :meth:`!list.append` *method* (we will see more about methods later):: >>> cubes.append(216) # add the cube of 6 >>> cubes.append(7 ** 3) # and the cube of 7 diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index a1bdcc236e82b7..64b951da3fd5b8 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1347,7 +1347,7 @@ complete list of changes, or look through the SVN logs for all the details. :func:`input` function to allow opening files in binary or :term:`universal newlines` mode. Another new parameter, *openhook*, lets you use a function other than :func:`open` to open the input files. Once you're iterating over - the set of files, the :class:`FileInput` object's new :meth:`fileno` returns + the set of files, the :class:`FileInput` object's new :meth:`~fileinput.fileno` returns the file descriptor for the currently opened file. (Contributed by Georg Brandl.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 257025da91a7ed..48a0e621baad02 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1798,7 +1798,7 @@ Standard Library * :func:`importlib.resources.path` * The :func:`locale.getdefaultlocale` function is deprecated and will be - removed in Python 3.13. Use :func:`locale.setlocale`, + removed in Python 3.15. Use :func:`locale.setlocale`, :func:`locale.getpreferredencoding(False) ` and :func:`locale.getlocale` functions instead. (Contributed by Victor Stinner in :gh:`90817`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index dc0cc82475a2e4..36d12feebea79b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1360,7 +1360,6 @@ Other modules: APIs: * :class:`!configparser.LegacyInterpolation` (:gh:`90765`) -* :func:`locale.getdefaultlocale` (:gh:`90817`) * ``locale.resetlocale()`` (:gh:`90817`) * :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) * :func:`!unittest.findTestCases` (:gh:`50096`) @@ -1430,6 +1429,17 @@ and will be removed in Python 3.14. * The ``co_lnotab`` attribute of code objects. +Pending Removal in Python 3.15 +------------------------------ + +The following APIs have been deprecated +and will be removed in Python 3.15. + +APIs: + +* :func:`locale.getdefaultlocale` (:gh:`90817`) + + Pending Removal in Future Versions ---------------------------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 358bd1747cd2c5..1053aa5729ede4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -65,6 +65,15 @@ Summary -- Release highlights .. PEP-sized items next. +Important deprecations, removals or restrictions: + +* :ref:`PEP 594 `: The remaining 19 "dead batteries" + have been removed from the standard library: + :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, + :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, + :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, + :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. + * :pep:`602` ("Annual Release Cycle for Python") has been updated: * Python 3.9 - 3.12 have one and a half years of full support, @@ -72,6 +81,7 @@ Summary -- Release highlights * Python 3.13 and later have two years of full support, followed by three years of security fixes. + New Features ============ @@ -106,6 +116,10 @@ Other Language Changes the file is not accessible. (Contributed by Moonsik Park in :gh:`82367`.) +* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block + is rejected when the global is used in the :keyword:`else` block. + (Contributed by Irit Katriel in :gh:`111123`.) + New Modules =========== @@ -486,6 +500,13 @@ Pending Removal in Python 3.15 rarely used. No direct replacement exists. *Anything* is better than CGI to interface a web server with a request handler. +* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 + and originally planned for removal in Python 3.13 (:gh:`90817`), + but removal has been postponed to Python 3.15. + Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and + :func:`locale.getlocale()` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + * :class:`typing.NamedTuple`: * The undocumented keyword argument syntax for creating NamedTuple classes @@ -598,10 +619,6 @@ although there is currently no date scheduled for their removal. `_ for migration advice. -* :func:`locale.getdefaultlocale`: use :func:`locale.setlocale()`, - :func:`locale.getencoding()` and :func:`locale.getlocale()` instead - (:gh:`90817`) - * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. @@ -686,184 +703,213 @@ although there is currently no date scheduled for their removal. Removed ======= -* :pep:`594`: Remove the :mod:`!telnetlib` module, deprecated in Python 3.11: - use the projects `telnetlib3 `_ or - `Exscript `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) +.. _whatsnew313-pep594: -* Remove the ``2to3`` program and the :mod:`!lib2to3` module, - deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104780`.) +PEP 594: dead batteries +----------------------- -* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, - are now removed. The items in those namespaces can be imported directly - from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) +* :pep:`594` removed 19 modules from the standard library, + deprecated in Python 3.11: -* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, - deprecated in Python 3.11. - Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. - (Contributed by Hugo van Kemenade in :gh:`104804`.) + * :mod:`!aifc`. + (Contributed by Victor Stinner in :gh:`104773`.) -* Remove support for using :class:`pathlib.Path` objects as context managers. - This functionality was deprecated and made a no-op in Python 3.9. + * :mod:`!audioop`. + (Contributed by Victor Stinner in :gh:`104773`.) -* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, - deprecated in the docstring since Python 3.2, - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104886`.) + * :mod:`!chunk`. + (Contributed by Victor Stinner in :gh:`104773`.) -* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, - deprecated in docs since Python 3.1 - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104876`.) + * :mod:`!cgi` and :mod:`!cgitb`. -* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: + * ``cgi.FieldStorage`` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or `multipart + `__ PyPI project for ``POST`` and + ``PUT``. - * :func:`!unittest.findTestCases` - * :func:`!unittest.makeSuite` - * :func:`!unittest.getTestCaseNames` + * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` + directly on the desired query string, except for ``multipart/form-data`` + input, which can be handled as described for ``cgi.parse_multipart()``. - Use :class:`~unittest.TestLoader` methods instead: + * ``cgi.parse_header()`` can be replaced with the functionality in the + :mod:`email` package, which implements the same MIME RFCs. For example, + with :class:`email.message.EmailMessage`:: - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` + from email.message import EmailMessage + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params - (Contributed by Hugo van Kemenade in :gh:`104835`.) + * ``cgi.parse_multipart()`` can be replaced with the functionality in the + :mod:`email` package (e.g. :class:`email.message.EmailMessage` and + :class:`email.message.Message`) which implements the same MIME RFCs, or + with the `multipart `__ PyPI project. -* :pep:`594`: Remove the :mod:`!cgi` and :mod:`!cgitb` modules, - deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!crypt` module and its private :mod:`!_crypt` extension. + The :mod:`hashlib` module is a potential replacement for certain use cases. + Otherwise, the following PyPI projects can be used: - * ``cgi.FieldStorage`` can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, and the - :mod:`email.message` module or `multipart - `__ PyPI project for ``POST`` and - ``PUT``. + * `bcrypt `_: + Modern password hashing for your software and your servers. + * `passlib `_: + Comprehensive password hashing framework supporting over 30 schemes. + * `argon2-cffi `_: + The secure Argon2 password hashing algorithm. + * `legacycrypt `_: + Wrapper to the POSIX crypt library call and associated functionality. - * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` - directly on the desired query string, except for ``multipart/form-data`` - input, which can be handled as described for ``cgi.parse_multipart()``. + (Contributed by Victor Stinner in :gh:`104773`.) - * ``cgi.parse_multipart()`` can be replaced with the functionality in the - :mod:`email` package (e.g. :class:`email.message.EmailMessage` and - :class:`email.message.Message`) which implements the same MIME RFCs, or - with the `multipart `__ PyPI project. + * :mod:`!imghdr`: use the projects + `filetype `_, + `puremagic `_, + or `python-magic `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) - * ``cgi.parse_header()`` can be replaced with the functionality in the - :mod:`email` package, which implements the same MIME RFCs. For example, - with :class:`email.message.EmailMessage`:: + * :mod:`!mailcap`. + The :mod:`mimetypes` module provides an alternative. + (Contributed by Victor Stinner in :gh:`104773`.) - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params + * :mod:`!msilib`. + (Contributed by Zachary Ware in :gh:`104773`.) - (Contributed by Victor Stinner in :gh:`104773`.) + * :mod:`!nis`. + (Contributed by Victor Stinner in :gh:`104773`.) -* :pep:`594`: Remove the :mod:`!sndhdr` module, deprecated in Python 3.11: use - the projects `filetype `_, `puremagic - `_, or `python-magic - `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * :mod:`!nntplib`: + the `PyPI nntplib project `_ + can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* :pep:`594`: Remove the :mod:`!pipes` module, deprecated in Python 3.11: - use the :mod:`subprocess` module instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * :mod:`!ossaudiodev`: use the + `pygame project `_ for audio playback. + (Contributed by Victor Stinner in :gh:`104780`.) -* :pep:`594`: Remove the :mod:`!ossaudiodev` module, deprecated in Python 3.11: - use the `pygame project `_ for audio playback. + * :mod:`!pipes`: use the :mod:`subprocess` module instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!sndhdr`: use the projects + `filetype `_, + `puremagic `_, or + `python-magic `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!spwd`: + the `python-pam project `_ + can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!sunau`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!telnetlib`, use the projects + `telnetlib3 `_ or + `Exscript `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!uu`: the :mod:`base64` module is a modern alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!xdrlib`. + (Contributed by Victor Stinner in :gh:`104773`.) + +2to3 +---- + +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. (Contributed by Victor Stinner in :gh:`104780`.) -* :pep:`594`: Remove the :mod:`!sunau` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) +configparser +------------ + +* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) -* :pep:`594`: Remove the :mod:`!mailcap` module, deprecated in Python 3.11. - The :mod:`mimetypes` module provides an alternative. - (Contributed by Victor Stinner in :gh:`104773`.) +locale +------ -* :pep:`594`: Remove the :mod:`!spwd` module, deprecated in Python 3.11: - the `python-pam project `_ can be used - instead. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: + use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) -* :pep:`594`: Remove the :mod:`!nntplib` module, deprecated in Python 3.11: - the `PyPI nntplib project `_ can be used - instead. - (Contributed by Victor Stinner in :gh:`104773`.) +logging +------- -* :pep:`594`: Remove the :mod:`!nis` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) +* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and + ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated + since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` + method, :meth:`!logging.LoggerAdapter.warning` method and + :func:`logging.warning` function. + (Contributed by Victor Stinner in :gh:`105376`.) + +pathlib +------- -* :pep:`594`: Remove the :mod:`!xdrlib` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. -* :pep:`594`: Remove the :mod:`!msilib` module, deprecated in Python 3.11. - (Contributed by Zachary Ware in :gh:`104773`.) +re +-- -* :pep:`594`: Remove the :mod:`!crypt` module and its private :mod:`!_crypt` - extension, deprecated in Python 3.11. - The :mod:`hashlib` module is a potential replacement for certain use cases. - Otherwise, the following PyPI projects can be used: +* Remove undocumented, never working, and deprecated ``re.template`` function + and ``re.TEMPLATE`` flag (and ``re.T`` alias). + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) - * `bcrypt `_: - Modern password hashing for your software and your servers. - * `passlib `_: - Comprehensive password hashing framework supporting over 30 schemes. - * `argon2-cffi `_: - The secure Argon2 password hashing algorithm. - * `legacycrypt `_: - Wrapper to the POSIX crypt library call and associated functionality. +tkinter +------- - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The + third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) -* :pep:`594`: Remove the :mod:`!uu` module, deprecated in Python 3.11: - the :mod:`base64` module is a modern alternative. - (Contributed by Victor Stinner in :gh:`104773`.) +turtle +------ -* :pep:`594`: Remove the :mod:`!aifc` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, + deprecated in docs since Python 3.1 + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) -* :pep:`594`: Remove the :mod:`!audioop` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) +typing +------ -* :pep:`594`: Remove the :mod:`!chunk` module, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104773`.) +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) * Remove support for the keyword-argument method of creating :class:`typing.TypedDict` types, deprecated in Python 3.11. (Contributed by Tomas Roun in :gh:`104786`.) -* :pep:`594`: Remove the :mod:`!imghdr` module, deprecated in Python 3.11: - use the projects - `filetype `_, - `puremagic `_, - or `python-magic `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) +unittest +-------- -* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` - method, deprecated in Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104992`.) +* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: -* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The - third-party Tix library which the module wrapped is unmaintained. - (Contributed by Zachary Ware in :gh:`75552`.) + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` -* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and - ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros - ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were - added in Python 3.8 and the old macros were deprecated in Python 3.11. - (Contributed by Irit Katriel in :gh:`105111`.) + Use :class:`~unittest.TestLoader` methods instead: -* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: - use ``locale.setlocale(locale.LC_ALL, "")`` instead. - (Contributed by Victor Stinner in :gh:`104783`.) + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) + (Contributed by Hugo van Kemenade in :gh:`104835`.) + +* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) + +urllib +------ * Remove *cafile*, *capath* and *cadefault* parameters of the :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the @@ -873,15 +919,23 @@ Removed certificates for you. (Contributed by Victor Stinner in :gh:`105382`.) +webbrowser +---------- + +* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, + deprecated in Python 3.11. + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) + * Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. Use :attr:`webbrowser.MacOSXOSAScript.name ` attribute instead. (Contributed by Nikita Sobolev in :gh:`105546`.) -* Remove undocumented, never working, and deprecated ``re.template`` function - and ``re.TEMPLATE`` flag (and ``re.T`` alias). - (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) +Others +------ +* None yet Porting to Python 3.13 ====================== @@ -889,34 +943,15 @@ Porting to Python 3.13 This section lists previously described changes and other bugfixes that may require changes to your code. -* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` - were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` - and ``Py_TRASHCAN_END``. - - A tp_dealloc function that has the old macros, such as:: - - static void - mytype_dealloc(mytype *p) - { - PyObject_GC_UnTrack(p); - Py_TRASHCAN_SAFE_BEGIN(p); - ... - Py_TRASHCAN_SAFE_END - } - - should migrate to the new macros as follows:: +Changes in the Python API +------------------------- - static void - mytype_dealloc(mytype *p) - { - PyObject_GC_UnTrack(p); - Py_TRASHCAN_BEGIN(p, mytype_dealloc) - ... - Py_TRASHCAN_END - } - - Note that ``Py_TRASHCAN_BEGIN`` has a second argument which - should be the deallocation function it is in. +* :meth:`!tkinter.Text.count` now always returns an integer if one or less + counting options are specified. + Previously it could return a single count as a 1-tuple, an integer (only if + option ``"update"`` was specified) or ``None`` if no items found. + The result is now the same if ``wantobjects`` is set to ``0``. + (Contributed by Serhiy Storchaka in :gh:`97928`.) Build Changes @@ -938,8 +973,9 @@ Build Changes library, GCC built-in atomic functions, or MSVC interlocked intrinsics. * The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, - ``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions - are now built with the :ref:`limited C API `. + ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the + :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) @@ -1068,6 +1104,9 @@ New Features limited C API. (Contributed by Victor Stinner in :gh:`85283`.) +* Add :c:func:`PyUnicode_AsUTF8` function to the limited C API. + (Contributed by Victor Stinner in :gh:`111089`.) + Porting to Python 3.13 ---------------------- @@ -1108,6 +1147,41 @@ Porting to Python 3.13 are now undefined by ````. (Contributed by Victor Stinner in :gh:`85283`.) +* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` + were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. + + A tp_dealloc function that has the old macros, such as:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_SAFE_BEGIN(p); + ... + Py_TRASHCAN_SAFE_END + } + + should migrate to the new macros as follows:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_BEGIN(p, mytype_dealloc) + ... + Py_TRASHCAN_END + } + + Note that ``Py_TRASHCAN_BEGIN`` has a second argument which + should be the deallocation function it is in. + +* The :c:func:`PyUnicode_AsUTF8` function now raises an exception if the string + contains embedded null characters. To accept embedded null characters and + truncate on purpose at the first null byte, + ``PyUnicode_AsUTF8AndSize(unicode, NULL)`` can be used instead. + (Contributed by Victor Stinner in :gh:`111089`.) + Deprecated ---------- @@ -1236,6 +1310,12 @@ Removed Configuration ` instead (:pep:`587`), added to Python 3.8. (Contributed by Victor Stinner in :gh:`105145`.) +* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and + ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros + ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were + added in Python 3.8 and the old macros were deprecated in Python 3.11. + (Contributed by Irit Katriel in :gh:`105111`.) + * Remove ``PyEval_InitThreads()`` and ``PyEval_ThreadsInitialized()`` functions, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` always creates the GIL: calling ``PyEval_InitThreads()`` did nothing and diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index b05ca3ef453816..64a40425ba7714 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -46,6 +46,7 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); +PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); /* Dictionary watchers */ diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 57834173490c99..ab885cf5ad03fd 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -7,3 +7,45 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); +/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in + base 256, and return a Python int with the same numeric value. + If n is 0, the integer is 0. Else: + If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; + else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the + LSB. + If is_signed is 0/false, view the bytes as a non-negative integer. + If is_signed is 1/true, view the bytes as a 2's-complement integer, + non-negative if bit 0x80 of the MSB is clear, negative if set. + Error returns: + + Return NULL with the appropriate exception set if there's not + enough memory to create the Python int. +*/ +PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( + const unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long + v to a base-256 integer, stored in array bytes. Normally return 0, + return -1 on error. + If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at + bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and + the LSB at bytes[n-1]. + If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes + are filled and there's nothing special about bit 0x80 of the MSB. + If is_signed is 1/true, bytes is filled with the 2's-complement + representation of v's value. Bit 0x80 of the MSB is the sign bit. + Error returns (-1): + + is_signed is 0 and v < 0. TypeError is set in this case, and bytes + isn't altered. + + n isn't big enough to hold the full mathematical value of v. For + example, if is_signed is 0 and there are more digits in the v than + fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of + being large enough to hold a sign bit. OverflowError is set in this + case, but bytes holds the least-significant n bytes of the true value. +*/ +PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, + unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* For use by the gcd function in mathmodule.c */ +PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 47536108a9665e..2a5251b3ecb02a 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -6,9 +6,27 @@ extern "C" { #endif +typedef struct _PyExecutorLinkListNode { + struct _PyExecutorObject *next; + struct _PyExecutorObject *previous; +} _PyExecutorLinkListNode; + + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define BLOOM_FILTER_WORDS 8 + +typedef struct _bloom_filter { + uint32_t bits[BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + typedef struct { uint8_t opcode; uint8_t oparg; + uint8_t valid; + uint8_t linked; + _PyBloomFilter bloom; + _PyExecutorLinkListNode links; } _PyVMData; typedef struct _PyExecutorObject { @@ -45,6 +63,14 @@ _PyOptimizer_BackEdge(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_ extern _PyOptimizerObject _PyOptimizer_Default; +void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *); +void _Py_ExecutorClear(_PyExecutorObject *); +void _Py_BloomFilter_Init(_PyBloomFilter *); +void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); +PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); +extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); + /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 859ab7178e920a..d200fa0622cef5 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -440,19 +440,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); -/* --- Manage the default encoding ---------------------------------------- */ - -/* Returns a pointer to the default encoding (UTF-8) of the - Unicode object unicode. - - Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation - in the unicodeobject. - - Use of this API is DEPRECATED since no size information can be - extracted from the returned data. -*/ - -PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); /* === Characters Type APIs =============================================== */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 47b5948f66343a..d01ef55de51f5d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -49,9 +49,6 @@ extern PyObject* _PyDict_NewPresized(Py_ssize_t minused); // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -// Export for '_socket' shared extension (Windows remove_unusable_flags()) -PyAPI_FUNC(PyObject*) _PyDict_Pop(PyObject *, PyObject *, PyObject *); - #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index bd448272e058ab..ec9703036c8ea5 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -58,26 +58,16 @@ typedef struct _PyInterpreterFrame { PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ - // NOTE: This is not necessarily the last instruction started in the given - // frame. Rather, it is the code unit *prior to* the *next* instruction. For - // example, it may be an inline CACHE entry, an instruction we just jumped - // over, or (in the case of a newly-created frame) a totally invalid value: - _Py_CODEUNIT *prev_instr; + _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ int stacktop; /* Offset of TOS from localsplus */ - /* The return_offset determines where a `RETURN` should go in the caller, - * relative to `prev_instr`. - * It is only meaningful to the callee, - * so it needs to be set in any CALL (to a Python function) - * or SEND (to a coroutine or generator). - * If there is no callee, then it is meaningless. */ - uint16_t return_offset; + uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ PyObject *localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->prev_instr - _PyCode_CODE(_PyFrame_GetCode(IF)))) + ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { assert(PyCode_Check(f->f_executable)); @@ -134,7 +124,7 @@ _PyFrame_Initialize( frame->f_locals = locals; frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code) - 1; + frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; @@ -185,7 +175,7 @@ _PyFrame_IsIncomplete(_PyInterpreterFrame *frame) return true; } return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->prev_instr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; + frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; } static inline _PyInterpreterFrame * @@ -297,7 +287,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code); + frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; return frame; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 60d333ad7baa2e..fc27aad48b5831 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -215,6 +215,7 @@ struct _is { struct types_state types; struct callable_cache callable_cache; _PyOptimizerObject *optimizer; + _PyExecutorObject *executor_list_head; uint16_t optimizer_resume_threshold; uint16_t optimizer_backedge_threshold; uint32_t next_func_version; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3c253ed7ff556b..ddf79ab1d130e8 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -128,57 +128,11 @@ extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int); // Export for '_datetime' shared extension. PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); -// _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in -// base 256, and return a Python int with the same numeric value. -// If n is 0, the integer is 0. Else: -// If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; -// else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the -// LSB. -// If is_signed is 0/false, view the bytes as a non-negative integer. -// If is_signed is 1/true, view the bytes as a 2's-complement integer, -// non-negative if bit 0x80 of the MSB is clear, negative if set. -// Error returns: -// + Return NULL with the appropriate exception set if there's not -// enough memory to create the Python int. -// -// Export for '_multibytecodec' shared extension. -PyAPI_DATA(PyObject*) _PyLong_FromByteArray( - const unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -// _PyLong_AsByteArray: Convert the least-significant 8*n bits of long -// v to a base-256 integer, stored in array bytes. Normally return 0, -// return -1 on error. -// If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at -// bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and -// the LSB at bytes[n-1]. -// If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes -// are filled and there's nothing special about bit 0x80 of the MSB. -// If is_signed is 1/true, bytes is filled with the 2's-complement -// representation of v's value. Bit 0x80 of the MSB is the sign bit. -// Error returns (-1): -// + is_signed is 0 and v < 0. TypeError is set in this case, and bytes -// isn't altered. -// + n isn't big enough to hold the full mathematical value of v. For -// example, if is_signed is 0 and there are more digits in the v than -// fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of -// being large enough to hold a sign bit. OverflowError is set in this -// case, but bytes holds the least-significant n bytes of the true value. -// -// Export for '_struct' shared extension. -PyAPI_DATA(int) _PyLong_AsByteArray(PyLongObject* v, - unsigned char* bytes, size_t n, - int little_endian, int is_signed); - // _PyLong_Format: Convert the long to a string object with given base, // appending a base prefix of 0[box] if base is 2, 8 or 16. // Export for '_tkinter' shared extension. PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); -// For use by the math.gcd() function. -// Export for 'math' shared extension. -PyAPI_DATA(PyObject*) _PyLong_GCD(PyObject *, PyObject *); - // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 926c0041c34c28..e2ed9bad7833a5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -87,7 +87,7 @@ #define _POP_JUMP_IF_FALSE 359 #define _POP_JUMP_IF_TRUE 360 #define _JUMP_TO_TOP 361 -#define _SAVE_CURRENT_IP 362 +#define _SAVE_RETURN_OFFSET 362 #define _INSERT 363 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -654,7 +654,7 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case _SET_IP: return 0; - case _SAVE_CURRENT_IP: + case _SAVE_RETURN_OFFSET: return 0; case _EXIT_TRACE: return 0; @@ -1230,7 +1230,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case _SET_IP: return 0; - case _SAVE_CURRENT_IP: + case _SAVE_RETURN_OFFSET: return 0; case _EXIT_TRACE: return 0; @@ -1296,7 +1296,7 @@ struct opcode_macro_expansion { #define OPARG_CACHE_4 4 #define OPARG_TOP 5 #define OPARG_BOTTOM 6 -#define OPARG_SET_IP 7 +#define OPARG_SAVE_RETURN_OFFSET 7 #define OPCODE_METADATA_FMT(OP) (_PyOpcode_opcode_metadata[(OP)].instr_format) #define SAME_OPCODE_METADATA(OP1, OP2) \ @@ -1589,7 +1589,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_SAVE_CURRENT_IP] = { true, INSTR_FMT_IX, 0 }, + [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, }; @@ -1644,8 +1644,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 2, .uops = { { _SAVE_CURRENT_IP, 7, -1 }, { _POP_FRAME, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 3, .uops = { { LOAD_CONST, 0, 0 }, { _SAVE_CURRENT_IP, 7, -1 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 2, .uops = { { LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, @@ -1719,8 +1719,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, @@ -1812,7 +1812,7 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", - [_SAVE_CURRENT_IP] = "_SAVE_CURRENT_IP", + [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_INSERT] = "_INSERT", }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index c4acb00a4b291e..a6733362d3bfdb 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -58,6 +58,12 @@ extern "C" { #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +/* Values used in the oparg for RESUME */ +#define RESUME_AT_FUNC_START 0 +#define RESUME_AFTER_YIELD 1 +#define RESUME_AFTER_YIELD_FROM 2 +#define RESUME_AFTER_AWAIT 3 + #ifdef __cplusplus } diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index ec003a1dad2595..61e0150e89009c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -114,6 +114,9 @@ extern int _Py_LegacyLocaleDetected(int warn); // Export for 'readline' shared extension PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); +// Export for special main.c string compiling with source tracebacks +int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f2383b43fabaf1..ec026d42f6d10b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -87,7 +87,7 @@ typedef struct _Py_DebugOffsets { struct _interpreter_frame { off_t previous; off_t executable; - off_t prev_instr; + off_t instr_ptr; off_t localsplus; off_t owner; } interpreter_frame; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 574a3c1a9db66c..73fa5f553cb81b 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -58,7 +58,7 @@ extern PyTypeObject _PyExc_MemoryError; .interpreter_frame = { \ .previous = offsetof(_PyInterpreterFrame, previous), \ .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ }, \ @@ -177,6 +177,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .last_resort_memory_error = { \ _PyObject_HEAD_INIT(&_PyExc_MemoryError) \ + .args = (PyObject*)&_Py_SINGLETON(tuple_empty) \ }, \ }, \ }, \ diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index dee00715b3c51d..ee7b769ce5a6fc 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -443,17 +443,25 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String( PyObject *unicode /* Unicode object */ ); -/* Returns a pointer to the default encoding (UTF-8) of the - Unicode object unicode and the size of the encoded representation - in bytes stored in *size. - - In case of an error, no *size is set. - - This function caches the UTF-8 encoded string in the unicodeobject - and subsequent calls will return the same string. The memory is released - when the unicodeobject is deallocated. -*/ - +// Returns a pointer to the UTF-8 encoding of the Unicode object unicode. +// +// Raise an exception if the string contains embedded null characters. +// Use PyUnicode_AsUTF8AndSize() to accept embedded null characters. +// +// This function caches the UTF-8 encoded string in the Unicode object +// and subsequent calls will return the same string. The memory is released +// when the Unicode object is deallocated. +PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); + +// Returns a pointer to the UTF-8 encoding of the +// Unicode object unicode and the size of the encoded representation +// in bytes stored in `*size` (if size is not NULL). +// +// On error, `*size` is set to 0 (if size is not NULL). +// +// This function caches the UTF-8 encoded string in the Unicode object +// and subsequent calls will return the same string. The memory is released +// when the Unicode object is deallocated. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize( PyObject *unicode, diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 24238c4f5f998d..91be0decc41c42 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -54,16 +54,14 @@ def __repr__(self): async def __aenter__(self): if self._entered: raise RuntimeError( - f"TaskGroup {self!r} has been already entered") - self._entered = True - + f"TaskGroup {self!r} has already been entered") if self._loop is None: self._loop = events.get_running_loop() - self._parent_task = tasks.current_task(self._loop) if self._parent_task is None: raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') + self._entered = True return self diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 029c468739bf2d..30042abb3ad804 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -49,8 +49,9 @@ def when(self) -> Optional[float]: def reschedule(self, when: Optional[float]) -> None: """Reschedule the timeout.""" - assert self._state is not _State.CREATED if self._state is not _State.ENTERED: + if self._state is _State.CREATED: + raise RuntimeError("Timeout has not been entered") raise RuntimeError( f"Cannot change state of {self._state.value} Timeout", ) @@ -82,11 +83,14 @@ def __repr__(self) -> str: return f"" async def __aenter__(self) -> "Timeout": + if self._state is not _State.CREATED: + raise RuntimeError("Timeout has already been entered") + task = tasks.current_task() + if task is None: + raise RuntimeError("Timeout should be used inside a task") self._state = _State.ENTERED - self._task = tasks.current_task() + self._task = task self._cancelling = self._task.cancelling() - if self._task is None: - raise RuntimeError("Timeout should be used inside a task") self.reschedule(self._when) return self diff --git a/Lib/compileall.py b/Lib/compileall.py index d394156cedc4e7..9b53086bf41380 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -172,13 +172,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: diff --git a/Lib/doctest.py b/Lib/doctest.py index 46b4dd6769b063..f00d9358ffe10b 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1393,7 +1393,20 @@ def __run(self, test, compileflags, out): # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exception[:2]) + if issubclass(exception[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(f"{exception[0].__name__}:") + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: got += _exception_traceback(exception) diff --git a/Lib/idlelib/sidebar.py b/Lib/idlelib/sidebar.py index fb1084dbf3f18b..166c04342907f9 100644 --- a/Lib/idlelib/sidebar.py +++ b/Lib/idlelib/sidebar.py @@ -25,10 +25,9 @@ def get_end_linenumber(text): def get_displaylines(text, index): """Display height, in lines, of a logical line in a Tk text widget.""" - res = text.count(f"{index} linestart", - f"{index} lineend", - "displaylines") - return res[0] if res else 0 + return text.count(f"{index} linestart", + f"{index} lineend", + "displaylines") def get_widget_padding(widget): """Get the total padding of a Tk widget, including its border.""" diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index f5aba434fd4253..68ddfbeb4bc212 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1970,6 +1970,9 @@ def __eq__(self, other): return False return self._scope_id == getattr(other, '_scope_id', None) + def __reduce__(self): + return (self.__class__, (str(self),)) + @property def scope_id(self): """Identifier of a particular zone of the address's scope. diff --git a/Lib/linecache.py b/Lib/linecache.py index c1c988d9df436c..329a14053458b7 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,10 +5,8 @@ that name. """ -import functools import sys import os -import tokenize __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -82,6 +80,8 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + import tokenize + if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) @@ -176,11 +176,13 @@ def lazycache(filename, module_globals): get_source = getattr(loader, 'get_source', None) if name and get_source: - get_lines = functools.partial(get_source, name) + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) cache[filename] = (get_lines,) return True return False + def _register_code(code, string, name): cache[code] = ( len(string), diff --git a/Lib/locale.py b/Lib/locale.py index 55c819ca80a160..e0cb4c5449d556 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -541,12 +541,14 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ import warnings - warnings.warn( - "Use setlocale(), getencoding() and getlocale() instead", - DeprecationWarning, stacklevel=2 - ) + warnings._deprecated( + "locale.getdefaultlocale", + "{name!r} is deprecated and slated for removal in Python {remove}. " + "Use setlocale(), getencoding() and getlocale() instead.", + remove=(3, 15)) return _getdefaultlocale(envvars) + def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): try: # check if it's supported by the _locale module diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 5c1c71ecec2805..e3eecc3b6d73e3 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -124,13 +124,13 @@ def _compile_pattern_lines(pattern_lines, case_sensitive): elif part == '*': part = r'.+' elif part == '**\n': - # '**/' component: we use '[\s\S]' rather than '.' so that path + # '**/' component: we use '(?s:.)' rather than '.' so that path # separators (i.e. newlines) are matched. The trailing '^' ensures # we terminate after a path separator (i.e. on a new line). - part = r'[\s\S]*^' + part = r'(?s:.)*^' elif part == '**': # '**' component. - part = r'[\s\S]*' + part = r'(?s:.)*' elif '**' in part: raise ValueError("Invalid pattern: '**' can only be an entire path component") else: diff --git a/Lib/socket.py b/Lib/socket.py index 321fcda51505c1..5f0a1f40e25b94 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -702,16 +702,15 @@ def readinto(self, b): self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - if e.errno in _blocking_errnos: - return None - raise + try: + return self._sock.recv_into(b) + except timeout: + self._timeout_occurred = True + raise + except error as e: + if e.errno in _blocking_errnos: + return None + raise def write(self, b): """Write the given bytes or bytearray object *b* to the socket diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 152b558808e66a..87b926db0686ce 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -161,8 +161,7 @@ def __init__(self, **kwargs) -> None: self.forever = False self.header = False self.failfast = False - self.match_tests = None - self.ignore_tests = None + self.match_tests = [] self.pgo = False self.pgo_extended = False self.worker_json = None @@ -183,6 +182,20 @@ def error(self, message): super().error(message + "\nPass -h or --help for complete help.") +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + items.append((value, self.const)) + + +class FromFileFilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + with open(value, encoding='utf-8') as fp: + for line in fp: + items.append((line.strip(), self.const)) + + def _create_parser(): # Set prog to prevent the uninformative "__main__.py" from displaying in # error messages when using "python -m test ...". @@ -192,6 +205,7 @@ def _create_parser(): epilog=EPILOG, add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.set_defaults(match_tests=[]) # Arguments with this clause added to its help are described further in # the epilog's "Additional option details" section. @@ -251,17 +265,19 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', action='append', + dest='match_tests', action=FilterAction, const=True, help='match test cases and methods with glob pattern PAT') group.add_argument('-i', '--ignore', metavar='PAT', - dest='ignore_tests', action='append', + dest='match_tests', action=FilterAction, const=False, help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', - dest='match_filename', + dest='match_tests', + action=FromFileFilterAction, const=True, help='similar to --match but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', - dest='ignore_filename', + dest='match_tests', + action=FromFileFilterAction, const=False, help='similar to --matchfile but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', @@ -482,18 +498,6 @@ def _parse_args(args, **kwargs): print("WARNING: Disable --verbose3 because it's incompatible with " "--huntrleaks: see http://bugs.python.org/issue27103", file=sys.stderr) - if ns.match_filename: - if ns.match_tests is None: - ns.match_tests = [] - with open(ns.match_filename) as fp: - for line in fp: - ns.match_tests.append(line.strip()) - if ns.ignore_filename: - if ns.ignore_tests is None: - ns.ignore_tests = [] - with open(ns.ignore_filename) as fp: - for line in fp: - ns.ignore_tests.append(line.strip()) if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py new file mode 100644 index 00000000000000..817624d79e9263 --- /dev/null +++ b/Lib/test/libregrtest/filter.py @@ -0,0 +1,72 @@ +import itertools +import operator +import re + + +# By default, don't filter tests +_test_matchers = () +_test_patterns = () + + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + result = False + for matcher, result in reversed(_test_matchers): + if matcher(test.id()): + return result + return not result + + +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def set_match_tests(patterns): + global _test_matchers, _test_patterns + + if not patterns: + _test_matchers = () + _test_patterns = () + else: + itemgetter = operator.itemgetter + patterns = tuple(patterns) + if patterns != _test_patterns: + _test_matchers = [ + (_compile_match_function(map(itemgetter(0), it)), result) + for result, it in itertools.groupby(patterns, itemgetter(1)) + ] + _test_patterns = patterns + + +def _compile_match_function(patterns): + patterns = list(patterns) + + if all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect_cmd utility only uses such full test identifiers. + return set(patterns).__contains__ + else: + import fnmatch + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id, regex_match=regex_match): + if regex_match(test_id): + # The regex matches the whole identifier, for example + # 'test.test_os.FileTests.test_access'. + return True + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + return match_test_regex diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index caf2872cfc9a18..78343775bc5b99 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -4,8 +4,9 @@ from test import support +from .filter import match_test, set_match_tests from .utils import ( - StrPath, TestName, TestTuple, TestList, FilterTuple, + StrPath, TestName, TestTuple, TestList, TestFilter, abs_module_name, count, printlist) @@ -79,15 +80,14 @@ def _list_cases(suite): if isinstance(test, unittest.TestSuite): _list_cases(test) elif isinstance(test, unittest.TestCase): - if support.match_test(test): + if match_test(test): print(test.id()) def list_cases(tests: TestTuple, *, - match_tests: FilterTuple | None = None, - ignore_tests: FilterTuple | None = None, + match_tests: TestFilter | None = None, test_dir: StrPath | None = None): support.verbose = False - support.set_match_tests(match_tests, ignore_tests) + set_match_tests(match_tests) skipped = [] for test_name in tests: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cb60d5af732b43..9b86548c89fb2e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,7 +19,7 @@ from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME from .utils import ( - StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, + StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, @@ -78,14 +78,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): and ns._add_python_opts) # Select tests - if ns.match_tests: - self.match_tests: FilterTuple | None = tuple(ns.match_tests) - else: - self.match_tests = None - if ns.ignore_tests: - self.ignore_tests: FilterTuple | None = tuple(ns.ignore_tests) - else: - self.ignore_tests = None + self.match_tests: TestFilter = ns.match_tests self.exclude: bool = ns.exclude self.fromfile: StrPath | None = ns.fromfile self.starting_test: TestName | None = ns.start @@ -129,14 +122,19 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Randomize self.randomize: bool = ns.randomize - self.random_seed: int | None = ( - ns.random_seed - if ns.random_seed is not None - else random.getrandbits(32) - ) - if 'SOURCE_DATE_EPOCH' in os.environ: + if ('SOURCE_DATE_EPOCH' in os.environ + # don't use the variable if empty + and os.environ['SOURCE_DATE_EPOCH'] + ): self.randomize = False - self.random_seed = None + # SOURCE_DATE_EPOCH should be an integer, but use a string to not + # fail if it's not integer. random.seed() accepts a string. + # https://reproducible-builds.org/docs/source-date-epoch/ + self.random_seed: int | str = os.environ['SOURCE_DATE_EPOCH'] + elif ns.random_seed is None: + self.random_seed = random.getrandbits(32) + else: + self.random_seed = ns.random_seed # tests self.first_runtests: RunTests | None = None @@ -384,7 +382,7 @@ def finalize_tests(self, tracer): def display_summary(self): duration = time.perf_counter() - self.logger.start_time - filtered = bool(self.match_tests) or bool(self.ignore_tests) + filtered = bool(self.match_tests) # Total duration print() @@ -402,7 +400,6 @@ def create_run_tests(self, tests: TestTuple): fail_fast=self.fail_fast, fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, match_tests_dict=None, rerun=False, forever=self.forever, @@ -441,7 +438,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources, self.python_cmd) - print("Using random seed", self.random_seed) + print("Using random seed:", self.random_seed) runtests = self.create_run_tests(selected) self.first_runtests = runtests @@ -655,7 +652,6 @@ def main(self, tests: TestList | None = None): elif self.want_list_cases: list_cases(selected, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index d6b0d5ad383a5b..8bfd3665ac93d5 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -2,13 +2,35 @@ import json from typing import Any -from test.support import TestStats - from .utils import ( StrJSON, TestName, FilterTuple, format_duration, normalize_test_name, print_warning) +@dataclasses.dataclass(slots=True) +class TestStats: + tests_run: int = 0 + failures: int = 0 + skipped: int = 0 + + @staticmethod + def from_unittest(result): + return TestStats(result.testsRun, + len(result.failures), + len(result.skipped)) + + @staticmethod + def from_doctest(results): + return TestStats(results.attempted, + results.failed, + results.skipped) + + def accumulate(self, stats): + self.tests_run += stats.tests_run + self.failures += stats.failures + self.skipped += stats.skipped + + # Avoid enum.Enum to reduce the number of imports when tests are run class State: PASSED = "PASSED" diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3708078ff0bf3a..1feb43f8c074db 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,8 +1,7 @@ import sys -from test.support import TestStats from .runtests import RunTests -from .result import State, TestResult +from .result import State, TestResult, TestStats from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 16f8331abd32f9..ab03cb54d6122e 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -261,7 +261,7 @@ def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> Ru kwargs = {} if match_tests: - kwargs['match_tests'] = match_tests + kwargs['match_tests'] = [(test, True) for test in match_tests] if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 4da312db4cb02e..bfed1b4a2a5817 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -8,7 +8,7 @@ from test import support from .utils import ( - StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) + StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict) class JsonFileType: @@ -72,8 +72,7 @@ class RunTests: tests: TestTuple fail_fast: bool fail_env_changed: bool - match_tests: FilterTuple | None - ignore_tests: FilterTuple | None + match_tests: TestFilter match_tests_dict: FilterDict | None rerun: bool forever: bool @@ -91,7 +90,7 @@ class RunTests: use_resources: tuple[str, ...] python_cmd: tuple[str, ...] | None randomize: bool - random_seed: int | None + random_seed: int | str json_file: JsonFile | None def copy(self, **override): diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 793347f60ad93c..97edba9f87d7f9 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -8,6 +8,7 @@ from test import support from test.support.os_helper import TESTFN_UNDECODABLE, FS_NONASCII +from .filter import set_match_tests from .runtests import RunTests from .utils import ( setup_unraisable_hook, setup_threading_excepthook, fix_umask, @@ -92,11 +93,11 @@ def setup_tests(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended - support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + set_match_tests(runtests.match_tests) if runtests.use_junit: support.junit_xml_list = [] - from test.support.testresult import RegressionTestResult + from .testresult import RegressionTestResult RegressionTestResult.USE_XML = True else: support.junit_xml_list = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 0304f858edf42c..ad75ef54a8c3f8 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -9,13 +9,14 @@ import unittest from test import support -from test.support import TestStats from test.support import threading_helper -from .result import State, TestResult +from .filter import match_test +from .result import State, TestResult, TestStats from .runtests import RunTests from .save_env import saved_test_environment from .setup import setup_tests +from .testresult import get_test_runner from .utils import ( TestName, clear_caches, remove_testfn, abs_module_name, print_warning) @@ -33,7 +34,47 @@ def run_unittest(test_mod): print(error, file=sys.stderr) if loader.errors: raise Exception("errors while loading tests") - return support.run_unittest(tests) + _filter_suite(tests, match_test) + return _run_suite(tests) + +def _filter_suite(suite, pred): + """Recursively filter test cases in a suite based on a predicate.""" + newtests = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _filter_suite(test, pred) + newtests.append(test) + else: + if pred(test): + newtests.append(test) + suite._tests = newtests + +def _run_suite(suite): + """Run tests from a unittest.TestSuite-derived class.""" + runner = get_test_runner(sys.stdout, + verbosity=support.verbose, + capture_output=(support.junit_xml_list is not None)) + + result = runner.run(suite) + + if support.junit_xml_list is not None: + support.junit_xml_list.append(result.get_xml_element()) + + if not result.testsRun and not result.skipped and not result.errors: + raise support.TestDidNotRun + if not result.wasSuccessful(): + stats = TestStats.from_unittest(result) + if len(result.errors) == 1 and not result.failures: + err = result.errors[0][1] + elif len(result.failures) == 1 and not result.errors: + err = result.failures[0][1] + else: + err = "multiple errors occurred" + if not support.verbose: err += "; run in verbose mode for details" + errors = [(str(tc), exc_str) for tc, exc_str in result.errors] + failures = [(str(tc), exc_str) for tc, exc_str in result.failures] + raise support.TestFailedWithDetails(err, errors, failures, stats=stats) + return result def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: diff --git a/Lib/test/support/testresult.py b/Lib/test/libregrtest/testresult.py similarity index 100% rename from Lib/test/support/testresult.py rename to Lib/test/libregrtest/testresult.py diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index aac8395da87845..bd4dce3400f0a9 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -52,6 +52,7 @@ TestList = list[TestName] # --match and --ignore options: list of patterns # ('*' joker character can be used) +TestFilter = list[tuple[TestName, bool]] FilterTuple = tuple[TestName, ...] FilterDict = dict[TestName, FilterTuple] diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index a9c8be0bb65d08..2eccfabc25223a 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -10,7 +10,7 @@ from .runtests import RunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( - StrPath, StrJSON, FilterTuple, + StrPath, StrJSON, TestFilter, get_temp_dir, get_work_dir, exit_timeout) @@ -73,7 +73,7 @@ def create_worker_process(runtests: RunTests, output_fd: int, def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] - match_tests: FilterTuple | None = runtests.match_tests + match_tests: TestFilter = runtests.match_tests json_file: JsonFile = runtests.json_file setup_test_dir(runtests.test_dir) @@ -81,7 +81,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: if runtests.rerun: if match_tests: - matching = "matching: " + ", ".join(match_tests) + matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result) print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) else: print(f"Re-running {test_name} in verbose mode", flush=True) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 21e8770ab31180..90fb1e670dfb38 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -19,8 +19,6 @@ import unittest import warnings -from .testresult import get_test_runner - __all__ = [ # globals @@ -34,7 +32,6 @@ "is_resource_enabled", "requires", "requires_freebsd_version", "requires_linux_version", "requires_mac_ver", "check_syntax_error", - "run_unittest", "run_doctest", "requires_gzip", "requires_bz2", "requires_lzma", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "requires_zlib", @@ -1118,176 +1115,6 @@ def requires_specialization(test): return unittest.skipUnless( _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) -def _filter_suite(suite, pred): - """Recursively filter test cases in a suite based on a predicate.""" - newtests = [] - for test in suite._tests: - if isinstance(test, unittest.TestSuite): - _filter_suite(test, pred) - newtests.append(test) - else: - if pred(test): - newtests.append(test) - suite._tests = newtests - -@dataclasses.dataclass(slots=True) -class TestStats: - tests_run: int = 0 - failures: int = 0 - skipped: int = 0 - - @staticmethod - def from_unittest(result): - return TestStats(result.testsRun, - len(result.failures), - len(result.skipped)) - - @staticmethod - def from_doctest(results): - return TestStats(results.attempted, - results.failed, - results.skipped) - - def accumulate(self, stats): - self.tests_run += stats.tests_run - self.failures += stats.failures - self.skipped += stats.skipped - - -def _run_suite(suite): - """Run tests from a unittest.TestSuite-derived class.""" - runner = get_test_runner(sys.stdout, - verbosity=verbose, - capture_output=(junit_xml_list is not None)) - - result = runner.run(suite) - - if junit_xml_list is not None: - junit_xml_list.append(result.get_xml_element()) - - if not result.testsRun and not result.skipped and not result.errors: - raise TestDidNotRun - if not result.wasSuccessful(): - stats = TestStats.from_unittest(result) - if len(result.errors) == 1 and not result.failures: - err = result.errors[0][1] - elif len(result.failures) == 1 and not result.errors: - err = result.failures[0][1] - else: - err = "multiple errors occurred" - if not verbose: err += "; run in verbose mode for details" - errors = [(str(tc), exc_str) for tc, exc_str in result.errors] - failures = [(str(tc), exc_str) for tc, exc_str in result.failures] - raise TestFailedWithDetails(err, errors, failures, stats=stats) - return result - - -# By default, don't filter tests -_match_test_func = None - -_accept_test_patterns = None -_ignore_test_patterns = None - - -def match_test(test): - # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) - - -def _is_full_match_test(pattern): - # If a pattern contains at least one dot, it's considered - # as a full test identifier. - # Example: 'test.test_os.FileTests.test_access'. - # - # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, ignore 'test_access*'. - return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) - - -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns - - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () - - accept_func = ignore_func = None - - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) - - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function - - -def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): - # Simple case: all patterns are full test identifier. - # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ - else: - import fnmatch - regex = '|'.join(map(fnmatch.translate, patterns)) - # The search *is* case sensitive on purpose: - # don't use flags=re.IGNORECASE - regex_match = re.compile(regex).match - - def match_test_regex(test_id): - if regex_match(test_id): - # The regex matches the whole identifier, for example - # 'test.test_os.FileTests.test_access'. - return True - else: - # Try to match parts of the test identifier. - # For example, split 'test.test_os.FileTests.test_access' - # into: 'test', 'test_os', 'FileTests' and 'test_access'. - return any(map(regex_match, test_id.split("."))) - - func = match_test_regex - - return patterns, func - - -def run_unittest(*classes): - """Run tests from unittest.TestCase-derived classes.""" - valid_types = (unittest.TestSuite, unittest.TestCase) - loader = unittest.TestLoader() - suite = unittest.TestSuite() - for cls in classes: - if isinstance(cls, str): - if cls in sys.modules: - suite.addTest(loader.loadTestsFromModule(sys.modules[cls])) - else: - raise ValueError("str arguments must be keys in sys.modules") - elif isinstance(cls, valid_types): - suite.addTest(cls) - else: - suite.addTest(loader.loadTestsFromTestCase(cls)) - _filter_suite(suite, match_test) - return _run_suite(suite) #======================================================================= # Check for the presence of docstrings. @@ -1309,38 +1136,6 @@ def _check_docstrings(): "test requires docstrings") -#======================================================================= -# doctest driver. - -def run_doctest(module, verbosity=None, optionflags=0): - """Run doctest on the given module. Return (#failures, #tests). - - If optional argument verbosity is not specified (or is None), pass - support's belief about verbosity on to doctest. Else doctest's - usual behavior is used (it searches sys.argv for -v). - """ - - import doctest - - if verbosity is None: - verbosity = verbose - else: - verbosity = None - - results = doctest.testmod(module, - verbose=verbosity, - optionflags=optionflags) - if results.failed: - stats = TestStats.from_doctest(results) - raise TestFailed(f"{results.failed} of {results.attempted} " - f"doctests failed", - stats=stats) - if verbose: - print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, results.attempted)) - return results - - #======================================================================= # Support for saving and restoring the imported modules. diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 6a0231f2859a62..7a18362b54e469 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -8,6 +8,8 @@ from asyncio import taskgroups import unittest +from test.test_asyncio.utils import await_without_task + # To prevent a warning "test altered the execution environment" def tearDownModule(): @@ -779,6 +781,49 @@ async def main(): await asyncio.create_task(main()) + async def test_taskgroup_already_entered(self): + tg = taskgroups.TaskGroup() + async with tg: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_double_enter(self): + tg = taskgroups.TaskGroup() + async with tg: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_finished(self): + tg = taskgroups.TaskGroup() + async with tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + # We still have to await coro to avoid a warning + await coro + + async def test_taskgroup_not_entered(self): + tg = taskgroups.TaskGroup() + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + # We still have to await coro to avoid a warning + await coro + + async def test_taskgroup_without_parent_task(self): + tg = taskgroups.TaskGroup() + with self.assertRaisesRegex(RuntimeError, "parent task"): + await await_without_task(tg.__aenter__()) + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + # We still have to await coro to avoid a warning + await coro + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index e9b59b953518b3..f54e79e4d8e600 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -5,11 +5,12 @@ import asyncio +from test.test_asyncio.utils import await_without_task + def tearDownModule(): asyncio.set_event_loop_policy(None) - class TimeoutTests(unittest.IsolatedAsyncioTestCase): async def test_timeout_basic(self): @@ -257,6 +258,51 @@ async def test_timeout_exception_cause (self): cause = exc.exception.__cause__ assert isinstance(cause, asyncio.CancelledError) + async def test_timeout_already_entered(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_double_enter(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_finished(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "finished"): + cm.reschedule(0.02) + + async def test_timeout_expired(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expired"): + cm.reschedule(0.02) + + async def test_timeout_expiring(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expiring"): + cm.reschedule(0.02) + + async def test_timeout_not_entered(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_without_task(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "task"): + await await_without_task(cm.__aenter__()) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 83fac4a26aff9e..18869b3290a8ae 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -612,3 +612,18 @@ def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, sock.family = family sock.gettimeout.return_value = 0.0 return sock + + +async def await_without_task(coro): + exc = None + def func(): + try: + for _ in coro.__await__(): + pass + except BaseException as err: + nonlocal exc + exc = err + asyncio.get_running_loop().call_soon(func) + await asyncio.sleep(0) + if exc is not None: + raise exc diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index ef744f6b97259c..82dea8a6d731ea 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -474,6 +474,12 @@ def test_base64_roundtrip(self, binary, newline): restored = binascii.a2b_base64(self.type2test(converted)) self.assertConversion(binary, converted, restored, newline=newline) + def test_c_contiguity(self): + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + binascii.b2a_hex(noncontig_writable) + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 96d34ab94c5981..c964b1efd577ba 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -153,6 +153,8 @@ class TupleSubclass(tuple): class DictSubclass(dict): pass +NONCONTIG_WRITABLE = memoryview(bytearray(b'noncontig'))[::-2] +NONCONTIG_READONLY = memoryview(b'noncontig')[::-2] class Unsigned_TestCase(unittest.TestCase): def test_b(self): @@ -837,6 +839,8 @@ def test_y_star(self): self.assertEqual(getargs_y_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_y_star(memoryview(b'memoryview')), b'memoryview') self.assertRaises(TypeError, getargs_y_star, None) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_READONLY) def test_y_hash(self): from _testcapi import getargs_y_hash @@ -846,6 +850,9 @@ def test_y_hash(self): self.assertRaises(TypeError, getargs_y_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_y_hash, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_y_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_READONLY) def test_w_star(self): # getargs_w_star() modifies first and last byte @@ -861,6 +868,17 @@ def test_w_star(self): self.assertEqual(getargs_w_star(memoryview(buf)), b'[emoryvie]') self.assertEqual(buf, bytearray(b'[emoryvie]')) self.assertRaises(TypeError, getargs_w_star, None) + self.assertRaises(TypeError, getargs_w_star, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_w_star, NONCONTIG_READONLY) + + def test_getargs_empty(self): + from _testcapi import getargs_empty + self.assertTrue(getargs_empty()) + self.assertRaises(TypeError, getargs_empty, 1) + self.assertRaises(TypeError, getargs_empty, 1, 2, 3) + self.assertRaises(TypeError, getargs_empty, a=1) + self.assertRaises(TypeError, getargs_empty, a=1, b=2) + self.assertRaises(TypeError, getargs_empty, 'x', 'y', 'z', a=1, b=2) class String_TestCase(unittest.TestCase): @@ -893,6 +911,8 @@ def test_s_star(self): self.assertEqual(getargs_s_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_s_star(memoryview(b'memoryview')), b'memoryview') self.assertRaises(TypeError, getargs_s_star, None) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_READONLY) def test_s_hash(self): from _testcapi import getargs_s_hash @@ -902,6 +922,9 @@ def test_s_hash(self): self.assertRaises(TypeError, getargs_s_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_s_hash, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_s_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_READONLY) def test_z(self): from _testcapi import getargs_z @@ -920,6 +943,8 @@ def test_z_star(self): self.assertEqual(getargs_z_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_z_star(memoryview(b'memoryview')), b'memoryview') self.assertIsNone(getargs_z_star(None)) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_READONLY) def test_z_hash(self): from _testcapi import getargs_z_hash @@ -929,6 +954,9 @@ def test_z_hash(self): self.assertRaises(TypeError, getargs_z_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_z_hash, memoryview(b'memoryview')) self.assertIsNone(getargs_z_hash(None)) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_READONLY) def test_es(self): from _testcapi import getargs_es @@ -1287,11 +1315,5 @@ def test_nonascii_keywords(self): parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) -class Test_testcapi(unittest.TestCase): - locals().update((name, getattr(_testcapi, name)) - for name in dir(_testcapi) - if name.startswith('test_') and name.endswith('_code')) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index b3f32d860eee03..ff34d85416e2df 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1098,46 +1098,6 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) - def test_sys_getobject(self): - getobject = _testcapi.sys_getobject - - self.assertIs(getobject(b'stdout'), sys.stdout) - with support.swap_attr(sys, '\U0001f40d', 42): - self.assertEqual(getobject('\U0001f40d'.encode()), 42) - - self.assertIs(getobject(b'nonexisting'), AttributeError) - self.assertIs(getobject(b'\xff'), AttributeError) - # CRASHES getobject(NULL) - - def test_sys_setobject(self): - setobject = _testcapi.sys_setobject - - value = ['value'] - value2 = ['value2'] - try: - self.assertEqual(setobject(b'newattr', value), 0) - self.assertIs(sys.newattr, value) - self.assertEqual(setobject(b'newattr', value2), 0) - self.assertIs(sys.newattr, value2) - self.assertEqual(setobject(b'newattr', NULL), 0) - self.assertFalse(hasattr(sys, 'newattr')) - self.assertEqual(setobject(b'newattr', NULL), 0) - finally: - with contextlib.suppress(AttributeError): - del sys.newattr - try: - self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) - self.assertIs(getattr(sys, '\U0001f40d'), value) - self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) - self.assertFalse(hasattr(sys, '\U0001f40d')) - finally: - with contextlib.suppress(AttributeError): - delattr(sys, '\U0001f40d') - - with self.assertRaises(UnicodeDecodeError): - setobject(b'\xff', value) - # CRASHES setobject(NULL, value) - @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): @@ -2159,7 +2119,7 @@ def test_gilstate_matches_current(self): class Test_testcapi(unittest.TestCase): locals().update((name, getattr(_testcapi, name)) for name in dir(_testcapi) - if name.startswith('test_') and not name.endswith('_code')) + if name.startswith('test_')) # Suppress warning from PyUnicode_FromUnicode(). @warnings_helper.ignore_warnings(category=DeprecationWarning) @@ -2489,6 +2449,67 @@ def get_first_executor(func): return None +class TestExecutorInvalidation(unittest.TestCase): + + def setUp(self): + self.old = _testinternalcapi.get_optimizer() + self.opt = _testinternalcapi.get_counter_optimizer() + _testinternalcapi.set_optimizer(self.opt) + + def tearDown(self): + _testinternalcapi.set_optimizer(self.old) + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range(1000): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor mutliple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def f(): + for i in range(1000): + pass + """), ns, ns) + f = ns['f'] + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + class TestUops(unittest.TestCase): def test_basic_loop(self): diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py new file mode 100644 index 00000000000000..b83a0a1604d06e --- /dev/null +++ b/Lib/test/test_capi/test_sys.py @@ -0,0 +1,150 @@ +import unittest +import contextlib +import sys +from test import support +from test.support import import_helper + +try: + import _testcapi +except ImportError: + _testcapi = None + +NULL = None + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PySys_Audit() + # PySys_AuditTuple() + + maxDiff = None + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_sys_getobject(self): + # Test PySys_GetObject() + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_sys_setobject(self): + # Test PySys_SetObject() + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_sys_getxoptions(self): + # Test PySys_GetXOptions() + getxoptions = _testcapi.sys_getxoptions + + self.assertIs(getxoptions(), sys._xoptions) + + xoptions = sys._xoptions + try: + sys._xoptions = 'non-dict' + self.assertEqual(getxoptions(), {}) + self.assertIs(getxoptions(), sys._xoptions) + + del sys._xoptions + self.assertEqual(getxoptions(), {}) + self.assertIs(getxoptions(), sys._xoptions) + finally: + sys._xoptions = xoptions + self.assertIs(getxoptions(), sys._xoptions) + + def _test_sys_formatstream(self, funname, streamname): + import_helper.import_module('ctypes') + from ctypes import pythonapi, c_char_p, py_object + func = getattr(pythonapi, funname) + func.argtypes = (c_char_p,) + + # Supports plain C types. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world')) + self.assertEqual(stream.getvalue(), 'Hello, world!') + + # Supports Python objects. + with support.captured_output(streamname) as stream: + func(b'Hello, %R!', py_object('world')) + self.assertEqual(stream.getvalue(), "Hello, 'world'!") + + # The total length is not limited. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*200)) + self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*200 + '!') + + def test_sys_formatstdout(self): + # Test PySys_FormatStdout() + self._test_sys_formatstream('PySys_FormatStdout', 'stdout') + + def test_sys_formatstderr(self): + # Test PySys_FormatStderr() + self._test_sys_formatstream('PySys_FormatStderr', 'stderr') + + def _test_sys_writestream(self, funname, streamname): + import_helper.import_module('ctypes') + from ctypes import pythonapi, c_char_p + func = getattr(pythonapi, funname) + func.argtypes = (c_char_p,) + + # Supports plain C types. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world')) + self.assertEqual(stream.getvalue(), 'Hello, world!') + + # There is a limit on the total length. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*100)) + self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*100 + '!') + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*200)) + out = stream.getvalue() + self.assertEqual(out[:20], 'Hello, worldworldwor') + self.assertEqual(out[-13:], '... truncated') + self.assertGreater(len(out), 1000) + + def test_sys_writestdout(self): + # Test PySys_WriteStdout() + self._test_sys_writestream('PySys_WriteStdout', 'stdout') + + def test_sys_writestderr(self): + # Test PySys_WriteStderr() + self._test_sys_writestream('PySys_WriteStderr', 'stderr') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index a73e669dda7ddc..8ab55902b1fd12 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -882,7 +882,10 @@ def test_asutf8(self): self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') - self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') + + # disallow embedded null characters + self.assertRaises(ValueError, unicode_asutf8, 'abc\0', 0) + self.assertRaises(ValueError, unicode_asutf8, 'abc\0def', 0) self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) @@ -906,7 +909,11 @@ def test_asutf8andsize(self): self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0) # CRASHES unicode_asutf8andsize(NULL, 0) + # CRASHES unicode_asutf8andsize_null(NULL, 0) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 614c6b3c3b5299..48754d5a63da3b 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -684,6 +684,27 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) + def test_source_lines_are_shown_when_running_source(self): + _, _, stderr = assert_python_failure("-c", "1/0") + expected_lines = [ + b'Traceback (most recent call last):', + b' File "", line 1, in ', + b' 1/0', + b' ~^~', + b'ZeroDivisionError: division by zero'] + self.assertEqual(stderr.splitlines(), expected_lines) + + def test_syntaxerror_does_not_crash(self): + script = "nonlocal x\n" + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script', script) + exitcode, stdout, stderr = assert_python_failure(script_name) + text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() + # It used to crash in https://github.com/python/cpython/issues/111132 + self.assertTrue(text.endswith( + 'SyntaxError: nonlocal declaration not allowed at module level\n', + ), text) + def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index c4452e38934cf8..df6e5e4b55f728 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1283,6 +1283,23 @@ def test_remove_redundant_nop_edge_case(self): def f(): a if (1 if b else c) else d + def test_global_declaration_in_except_used_in_else(self): + # See gh-111123 + code = textwrap.dedent("""\ + def f(): + try: + pass + %s Exception: + global a + else: + print(a) + """) + + g, l = {'a': 5}, {} + for kw in ("except", "except*"): + exec(code % kw, g, l); + + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 9cd92ad365c5a9..83a9532aecfac8 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -362,6 +362,29 @@ def test_strip_only(self): str(err, encoding=sys.getdefaultencoding()) ) + def test_strip_only_invalid(self): + fullpath = ["test", "build", "real", "path"] + path = os.path.join(self.directory, *fullpath) + os.makedirs(path) + script = script_helper.make_script(path, "test", "1 / 0") + bc = importlib.util.cache_from_source(script) + stripdir = os.path.join(self.directory, *(fullpath[:2] + ['fake'])) + compileall.compile_dir(path, quiet=True, stripdir=stripdir) + rc, out, err = script_helper.assert_python_failure(bc) + expected_not_in = os.path.join(self.directory, *fullpath[2:]) + self.assertIn( + path, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + expected_not_in, + str(err, encoding=sys.getdefaultencoding()) + ) + self.assertNotIn( + stripdir, + str(err, encoding=sys.getdefaultencoding()) + ) + def test_prepend_only(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 5d94ec7cae4706..3dad2567015e24 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -173,6 +173,15 @@ def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.gi_suspended) + def test_contextmanager_trap_no_yield(self): + @contextmanager + def whoo(): + if False: + yield + ctx = whoo() + with self.assertRaises(RuntimeError): + ctx.__enter__() + def test_contextmanager_trap_second_yield(self): @contextmanager def whoo(): @@ -186,6 +195,19 @@ def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.gi_suspended) + def test_contextmanager_non_normalised(self): + @contextmanager + def whoo(): + try: + yield + except RuntimeError: + raise SyntaxError + + ctx = whoo() + ctx.__enter__() + with self.assertRaises(SyntaxError): + ctx.__exit__(RuntimeError, None, None) + def test_contextmanager_except(self): state = [] @contextmanager @@ -265,6 +287,25 @@ def test_issue29692(): self.assertEqual(ex.args[0], 'issue29692:Unchained') self.assertIsNone(ex.__cause__) + def test_contextmanager_wrap_runtimeerror(self): + @contextmanager + def woohoo(): + try: + yield + except Exception as exc: + raise RuntimeError(f'caught {exc}') from exc + + with self.assertRaises(RuntimeError): + with woohoo(): + 1 / 0 + + # If the context manager wrapped StopIteration in a RuntimeError, + # we also unwrap it, because we can't tell whether the wrapping was + # done by the generator machinery or by the generator itself. + with self.assertRaises(StopIteration): + with woohoo(): + raise StopIteration + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): @@ -276,6 +317,7 @@ def decorate(func): @attribs(foo='bar') def baz(spam): """Whee!""" + yield return baz def test_contextmanager_attribs(self): @@ -332,8 +374,11 @@ def woohoo(a, *, b): def test_recursive(self): depth = 0 + ncols = 0 @contextmanager def woohoo(): + nonlocal ncols + ncols += 1 nonlocal depth before = depth depth += 1 @@ -347,6 +392,7 @@ def recursive(): recursive() recursive() + self.assertEqual(ncols, 10) self.assertEqual(depth, 0) diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 3056fe84dac5dd..27e8a767903777 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -83,8 +83,8 @@ def test_throw(self): for func, (cc, nc, _, _, _) in pr.stats.items(): if func[2] == "": - self.assertEqual(cc, 2) - self.assertEqual(nc, 2) + self.assertEqual(cc, 1) + self.assertEqual(nc, 1) class TestCommandLine(unittest.TestCase): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 6e12e82a7a0084..e5b08a3c47a901 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -3212,25 +3212,155 @@ def test_run_doctestsuite_multiple_times(): """ +def test_exception_with_note(note): + """ + >>> test_exception_with_note('Note') + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('Note') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('''Note + ... multiline + ... example''') + Traceback (most recent call last): + ValueError: Text + Note + multiline + example + + Different note will fail the test: + + >>> def f(x): + ... r''' + ... >>> exc = ValueError('message') + ... >>> exc.add_note('note') + ... >>> raise exc + ... Traceback (most recent call last): + ... ValueError: message + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 5, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + ValueError: message + wrong note + Got: + Traceback (most recent call last): + ... + ValueError: message + note + TestResults(failed=1, attempted=...) + """ + exc = ValueError('Text') + exc.add_note(note) + raise exc + + +def test_exception_with_multiple_notes(): + """ + >>> test_exception_with_multiple_notes() + Traceback (most recent call last): + ... + ValueError: Text + One + Two + """ + exc = ValueError('Text') + exc.add_note('One') + exc.add_note('Two') + raise exc + + +def test_syntax_error_with_note(cls, multiline=False): + """ + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + File "x.py", line 23 + bad syntax + SyntaxError: error + Note + + >>> test_syntax_error_with_note(IndentationError) + Traceback (most recent call last): + ... + IndentationError: error + Note + + >>> test_syntax_error_with_note(TabError, multiline=True) + Traceback (most recent call last): + ... + TabError: error + Note + Line + """ + exc = cls("error", ("x.py", 23, None, "bad syntax")) + exc.add_note('Note\nLine' if multiline else 'Note') + raise exc + + +def test_syntax_error_with_incorrect_expected_note(): + """ + >>> def f(x): + ... r''' + ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + ... >>> exc.add_note('note1') + ... >>> exc.add_note('note2') + ... >>> raise exc + ... Traceback (most recent call last): + ... SyntaxError: error + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + SyntaxError: error + wrong note + Got: + Traceback (most recent call last): + ... + SyntaxError: error + note1 + note2 + TestResults(failed=1, attempted=...) + """ + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(doctest)) tests.addTest(doctest.DocTestSuite()) return tests -def test_coverage(coverdir): - trace = import_helper.import_module('trace') - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], - trace=0, count=1) - tracer.run('test_main()') - r = tracer.results() - print('Writing coverage results...') - r.write_results(show_missing=True, summary=True, - coverdir=coverdir) - - if __name__ == '__main__': - if '-c' in sys.argv: - test_coverage('/tmp/doctest.cover') - else: - unittest.main() + unittest.main(module='test.test_doctest') diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index 25fa48c5ee217b..c9d973df0a2192 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -5,6 +5,7 @@ import unittest import sys import os.path +import zoneinfo class DateTimeTests(unittest.TestCase): @@ -142,13 +143,9 @@ def test_localtime_epoch_notz_daylight_false(self): t2 = utils.localtime(t0.replace(tzinfo=None)) self.assertEqual(t1, t2) - # XXX: Need a more robust test for Olson's tzdata - @unittest.skipIf(sys.platform.startswith('win'), - "Windows does not use Olson's TZ database") - @unittest.skipUnless(os.path.exists('/usr/share/zoneinfo') or - os.path.exists('/usr/lib/zoneinfo'), - "Can't find the Olson's TZ database") - @test.support.run_with_tz('Europe/Kiev') + @unittest.skipUnless("Europe/Kyiv" in zoneinfo.available_timezones(), + "Can't find a Kyiv timezone database") + @test.support.run_with_tz('Europe/Kyiv') def test_variable_tzname(self): t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4031c5ca76c705..7f1d5ee9322b45 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1791,6 +1791,20 @@ class TestException(MemoryError): gc_collect() + def test_memory_error_in_subinterp(self): + # gh-109894: subinterpreters shouldn't count on last resort memory error + # when MemoryError is raised through PyErr_NoMemory() call, + # and should preallocate memory errors as does the main interpreter. + # interp.static_objects.last_resort_memory_error.args + # should be initialized to empty tuple to avoid crash on attempt to print it. + code = f"""if 1: + import _testcapi + _testcapi.run_in_subinterp(\"[0]*{sys.maxsize}\") + exit(0) + """ + rc, _, err = script_helper.assert_python_ok("-c", code) + self.assertIn(b'MemoryError', err) + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 31ac6b1070a0a2..33eae8505d5d24 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -16,6 +16,7 @@ import shutil import stat import sys +import subprocess import time import types import tempfile @@ -35,7 +36,8 @@ from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python +from test.support import has_subprocess_support, SuppressCrashReport from test import support from . import inspect_fodder as mod @@ -4961,5 +4963,66 @@ def test_getsource_reload(self): self.assertInspectEqual(path, module) +class TestRepl(unittest.TestCase): + + def spawn_repl(self, *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): + """Run the Python REPL with the given arguments. + + kw is extra keyword args to pass to subprocess.Popen. Returns a Popen + object. + """ + + # To run the REPL without using a terminal, spawn python with the command + # line option '-i' and the process name set to ''. + # The directory of argv[0] must match the directory of the Python + # executable for the Popen() call to python to succeed as the directory + # path may be used by Py_GetPath() to build the default module search + # path. + stdin_fname = os.path.join(os.path.dirname(sys.executable), "") + cmd_line = [stdin_fname, '-E', '-i'] + cmd_line.extend(args) + + # Set TERM=vt100, for the rationale see the comments in spawn_python() of + # test.support.script_helper. + env = kw.setdefault('env', dict(os.environ)) + env['TERM'] = 'vt100' + return subprocess.Popen(cmd_line, + executable=sys.executable, + text=True, + stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + **kw) + + def run_on_interactive_mode(self, source): + """Spawn a new Python interpreter, pass the given + input source code from the stdin and return the + result back. If the interpreter exits non-zero, it + raises a ValueError.""" + + process = self.spawn_repl() + process.stdin.write(source) + output = kill_python(process) + + if process.returncode != 0: + raise ValueError("Process didn't exit properly.") + return output + + @unittest.skipIf(not has_subprocess_support, "test requires subprocess") + def test_getsource(self): + output = self.run_on_interactive_mode(textwrap.dedent("""\ + def f(): + print(0) + return 1 + 2 + + import inspect + print(f"The source is: <<<{inspect.getsource(f)}>>>") + """)) + + expected = "The source is: <<>>" + self.assertIn(expected, output) + + + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 022cf21a4709a2..4c80429684e525 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4396,11 +4396,11 @@ def test_check_encoding_warning(self): ''') proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) warnings = proc.err.splitlines() - self.assertEqual(len(warnings), 2) + self.assertEqual(len(warnings), 4) self.assertTrue( warnings[0].startswith(b":5: EncodingWarning: ")) self.assertTrue( - warnings[1].startswith(b":8: EncodingWarning: ")) + warnings[2].startswith(b":8: EncodingWarning: ")) def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 6f204948c9fc48..33a0f9894a32f9 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -4,6 +4,7 @@ """Unittest for ipaddress module.""" +import copy import unittest import re import contextlib @@ -542,11 +543,17 @@ def assertBadPart(addr, part): def test_pickle(self): self.pickle_test('2001:db8::') + self.pickle_test('2001:db8::%scope') def test_weakref(self): weakref.ref(self.factory('2001:db8::')) weakref.ref(self.factory('2001:db8::%scope')) + def test_copy(self): + addr = self.factory('2001:db8::%scope') + self.assertEqual(addr, copy.copy(addr)) + self.assertEqual(addr, copy.deepcopy(addr)) + class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index cd2faba1791c77..731299294e6877 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -463,6 +463,20 @@ def test_getbuffer(self): memio.close() self.assertRaises(ValueError, memio.getbuffer) + def test_getbuffer_empty(self): + memio = self.ioclass() + buf = memio.getbuffer() + self.assertEqual(bytes(buf), b"") + # Trying to change the size of the BytesIO while a buffer is exported + # raises a BufferError. + self.assertRaises(BufferError, memio.write, b'x') + buf2 = memio.getbuffer() + self.assertRaises(BufferError, memio.write, b'x') + buf.release() + self.assertRaises(BufferError, memio.write, b'x') + buf2.release() + memio.write(b'x') + def test_read1(self): buf = self.buftype("1234567890") self.assertEqual(self.ioclass(buf).read1(), buf) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 6ea7e7db2ce113..4d1766fb16ce9f 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -7,6 +7,7 @@ import itertools import pprint import random +import re import test.support import test.test_set import types @@ -535,7 +536,10 @@ def test_dataclass_with_repr(self): def test_dataclass_no_repr(self): dc = dataclass3() formatted = pprint.pformat(dc, width=10) - self.assertRegex(formatted, r"") + self.assertRegex( + formatted, + fr"<{re.escape(__name__)}.dataclass3 object at \w+>", + ) def test_recursive_dataclass(self): dc = dataclass4(None) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e65d9a89ff8a2f..0c39af05eb5ea8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -22,11 +22,13 @@ import textwrap import unittest from test import support -from test.support import os_helper, TestStats, without_optimizer +from test.support import os_helper, without_optimizer from test.libregrtest import cmdline from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils +from test.libregrtest.filter import set_match_tests, match_test +from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name if not support.has_subprocess_support: @@ -143,18 +145,26 @@ def test_header(self): self.assertTrue(ns.header) def test_randomize(self): - for opt in '-r', '--randomize': + for opt in ('-r', '--randomize'): with self.subTest(opt=opt): ns = self.parse_args([opt]) self.assertTrue(ns.randomize) with os_helper.EnvironmentVarGuard() as env: - env['SOURCE_DATE_EPOCH'] = '1' - + # with SOURCE_DATE_EPOCH + env['SOURCE_DATE_EPOCH'] = '1697839080' ns = self.parse_args(['--randomize']) regrtest = main.Regrtest(ns) self.assertFalse(regrtest.randomize) - self.assertIsNone(regrtest.random_seed) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + # without SOURCE_DATE_EPOCH + del env['SOURCE_DATE_EPOCH'] + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) @@ -184,34 +194,27 @@ def test_single(self): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") - def test_ignore(self): - for opt in '-i', '--ignore': + def test_match(self): + for opt in '-m', '--match': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.ignore_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', True)]) self.checkError([opt], 'expected one argument') - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: - print('matchfile1', file=fp) - print('matchfile2', file=fp) - - filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--ignorefile', filename]) - self.assertEqual(ns.ignore_tests, - ['matchfile1', 'matchfile2']) - - def test_match(self): - for opt in '-m', '--match': + for opt in '-i', '--ignore': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', False)]) self.checkError([opt], 'expected one argument') - ns = self.parse_args(['-m', 'pattern1', - '-m', 'pattern2']) - self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)]) + + ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)]) + + ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)]) self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "w") as fp: @@ -219,10 +222,13 @@ def test_match(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--matchfile', filename]) + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) + self.assertEqual(ns.match_tests, + [('match', True), ('matchfile1', True), ('matchfile2', True)]) + + ns = self.parse_args(['-i', 'match', '--ignorefile', filename]) self.assertEqual(ns.match_tests, - ['match', 'matchfile1', 'matchfile2']) + [('match', False), ('matchfile1', False), ('matchfile2', False)]) def test_failfast(self): for opt in '-G', '--failfast': @@ -388,7 +394,13 @@ def check_ci_mode(self, args, use_resources, rerun=True): # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API - regrtest = main.Regrtest(ns) + with os_helper.EnvironmentVarGuard() as env: + # Ignore SOURCE_DATE_EPOCH env var if it's set + if 'SOURCE_DATE_EPOCH' in env: + del env['SOURCE_DATE_EPOCH'] + + regrtest = main.Regrtest(ns) + self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -661,21 +673,26 @@ def list_regex(line_format, tests): state = f'{state} then {new_state}' self.check_line(output, f'Result: {state}', full=True) - def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) - randseed = int(match.group(1)) - self.assertTrue(0 <= randseed, randseed) - return randseed + def parse_random_seed(self, output: str) -> str: + match = self.regex_search(r'Using random seed: (.*)', output) + return match.group(1) def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' if 'stderr' not in kw: kw['stderr'] = subprocess.STDOUT + + env = kw.pop('env', None) + if env is None: + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + proc = subprocess.run(args, text=True, input=input, stdout=subprocess.PIPE, + env=env, **kw) if proc.returncode != exitcode: msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" @@ -751,7 +768,9 @@ def setUp(self): self.regrtest_args.append('-n') def check_output(self, output): - self.parse_random_seed(output) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) @@ -942,7 +961,7 @@ def test_random(self): test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test, + output = self.run_tests('-r', f'--randseed={randseed}', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -953,7 +972,32 @@ def test_random(self): # check that random.seed is used by default output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) - self.assertIsInstance(self.parse_random_seed(output), int) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + + # check SOURCE_DATE_EPOCH (integer) + timestamp = '1697839080' + env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, timestamp) + self.check_line(output, 'TESTRANDOM: 520') + + # check SOURCE_DATE_EPOCH (string) + env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, 'XYZ') + self.check_line(output, 'TESTRANDOM: 22') + + # check SOURCE_DATE_EPOCH (empty string): ignore the env var + env = dict(os.environ, SOURCE_DATE_EPOCH='') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) def test_fromfile(self): # test --fromfile @@ -2140,6 +2184,120 @@ def test_format_resources(self): format_resources((*ALL_RESOURCES, "tzdata")), 'resources: all,tzdata') + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + test_copy = Test('test.test_shutil.TestCopy.test_copy') + + # Test acceptance + with support.swap_attr(support, '_test_matchers', ()): + # match all + set_match_tests([]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match all using None + set_match_tests(None) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the full test identifier + set_match_tests([(test_access.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', True)]) + self.assertFalse(match_test(test_access)) + set_match_tests([('FileTests', True)]) + self.assertTrue(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), True), (test_chdir.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + set_match_tests([('test_access', True), ('DONTMATCH', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test rejection + with support.swap_attr(support, '_test_matchers', ()): + # match the full test identifier + set_match_tests([(test_access.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', False)]) + self.assertTrue(match_test(test_access)) + set_match_tests([('FileTests', False)]) + self.assertFalse(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), False), (test_chdir.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('test_access', False), ('DONTMATCH', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test mixed filters + with support.swap_attr(support, '_test_matchers', ()): + set_match_tests([('*test_os', False), ('test_access', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('*test_os', True), ('test_access', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 2ee5117bcd7bd4..7533376e015e73 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -184,7 +184,7 @@ def bar(x): p.stdin.write(user_input) user_input2 = dedent(""" import linecache - print(linecache.cache['']) + print(linecache.cache['-1']) """) p.stdin.write(user_input2) output = kill_python(p) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 06304dcb4ec7b8..d8ae7b75e18150 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1739,6 +1739,10 @@ def test_buffer_types(self): self.assertEqual(bio.read(), b'bar') bio.write(memoryview(b'baz')) self.assertEqual(bio.read(), b'baz') + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + bio.write(memoryview(noncontig_writable)) def test_error_types(self): bio = ssl.MemoryBIO() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 6d5353c22764df..88bc0fd4025a17 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -745,6 +745,7 @@ def test_windows_feature_macros(self): "PyUnicode_AsUCS4Copy", "PyUnicode_AsUTF16String", "PyUnicode_AsUTF32String", + "PyUnicode_AsUTF8", "PyUnicode_AsUTF8AndSize", "PyUnicode_AsUTF8String", "PyUnicode_AsUnicodeEscapeString", diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index a865df1082fba3..fe1a3675fced65 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1769,9 +1769,9 @@ def test_encoding_warning(self): cp = subprocess.run([sys.executable, "-Xwarn_default_encoding", "-c", code], capture_output=True) lines = cp.stderr.splitlines() - self.assertEqual(len(lines), 2, lines) + self.assertEqual(len(lines), 4, lines) self.assertTrue(lines[0].startswith(b":2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b":3: EncodingWarning: ")) + self.assertTrue(lines[2].startswith(b":3: EncodingWarning: ")) def _get_test_grp_name(): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 97de81677b10bc..c34b0e5e015702 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -547,111 +547,6 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') - def test_match_test(self): - class Test: - def __init__(self, test_id): - self.test_id = test_id - - def id(self): - return self.test_id - - test_access = Test('test.test_os.FileTests.test_access') - test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') - - # Test acceptance - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests([]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests([test_access.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(['test_os'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(['test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(['filetests'], None) - self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests'], None) - self.assertTrue(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - support.set_match_tests(['test_access', 'DONTMATCH'], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test rejection - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests(ignore_patterns=[]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests(None, [test_access.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(None, ['test_os']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(None, ['test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(None, ['filetests']) - self.assertTrue(support.match_test(test_access)) - support.set_match_tests(None, ['FileTests']) - self.assertFalse(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(None, ['*test_os.*.test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests(None, [test_access.id(), test_chdir.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - support.set_match_tests(None, ['test_access', 'DONTMATCH']) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): @@ -852,7 +747,6 @@ def test_copy_python_src_ignore(self): # precisionbigmemtest # bigaddrspacetest # requires_resource - # run_doctest # threading_cleanup # reap_threads # can_symlink diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index da213506151594..b16e1e7f70a24a 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1114,14 +1114,18 @@ def check(tracebacklimit, expected): traceback = [ b'Traceback (most recent call last):', b' File "", line 8, in ', + b' f2()', b' File "", line 6, in f2', + b' f1()', b' File "", line 4, in f1', + b' 1 / 0', + b' ~~^~~', b'ZeroDivisionError: division by zero' ] check(10, traceback) check(3, traceback) - check(2, traceback[:1] + traceback[2:]) - check(1, traceback[:1] + traceback[3:]) + check(2, traceback[:1] + traceback[3:]) + check(1, traceback[:1] + traceback[5:]) check(0, [traceback[-1]]) check(-1, [traceback[-1]]) check(1<<1000, traceback) diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 9e8936630de920..bb8adc8b555cb4 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -265,10 +265,6 @@ def g(p): f_ident = ident(f) g_ident = ident(g) self.check_events(g, [(1, 'call', g_ident), - (2, 'call', f_ident), - (2, 'return', f_ident), - # once more; the generator is being garbage collected - # and it will do a PY_THROW (2, 'call', f_ident), (2, 'return', f_ident), (1, 'return', g_ident), diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index f02169602e4925..df7dd36274df2e 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -9,6 +9,7 @@ import asyncio from test.support import import_helper import contextlib +import warnings support.requires_working_socket(module=True) @@ -2001,6 +2002,9 @@ def run_test(self, func, jumpFrom, jumpTo, expected, error=None, stack.enter_context(self.assertRaisesRegex(*error)) if warning is not None: stack.enter_context(self.assertWarnsRegex(*warning)) + else: + stack.enter_context(warnings.catch_warnings()) + warnings.simplefilter('error') func(output) sys.settrace(None) @@ -2064,6 +2068,40 @@ def test_jump_simple_backwards(output): output.append(1) output.append(2) + @jump_test(1, 4, [5], warning=(RuntimeWarning, unbound_locals)) + def test_jump_is_none_forwards(output): + x = None + if x is None: + output.append(3) + else: + output.append(5) + + @jump_test(6, 5, [3, 5, 6]) + def test_jump_is_none_backwards(output): + x = None + if x is None: + output.append(3) + else: + output.append(5) + output.append(6) + + @jump_test(2, 4, [5]) + def test_jump_is_not_none_forwards(output): + x = None + if x is not None: + output.append(3) + else: + output.append(5) + + @jump_test(6, 5, [5, 5, 6]) + def test_jump_is_not_none_backwards(output): + x = None + if x is not None: + output.append(3) + else: + output.append(5) + output.append(6) + @jump_test(3, 5, [2, 5], warning=(RuntimeWarning, unbound_locals)) def test_jump_out_of_block_forwards(output): for i in 1, 2: diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 0ae27610be6078..ca99caaf88b80d 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -425,6 +425,309 @@ def test_info_patchlevel(self): self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) +class BindTest(AbstractTkTest, unittest.TestCase): + + def setUp(self): + super().setUp() + root = self.root + self.frame = tkinter.Frame(self.root, class_='Test', + width=150, height=100) + self.frame.pack() + + def assertCommandExist(self, funcid): + self.assertEqual(_info_commands(self.root, funcid), (funcid,)) + + def assertCommandNotExist(self, funcid): + self.assertEqual(_info_commands(self.root, funcid), ()) + + def test_bind(self): + event = '' + f = self.frame + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + + funcid = f.bind(event, test1) + self.assertEqual(f.bind(), (event,)) + script = f.bind(event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = f.bind(event, test2, add=True) + script = f.bind(event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def test_unbind(self): + event = '' + f = self.frame + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + + funcid = f.bind(event, test1) + funcid2 = f.bind(event, test2, add=True) + + self.assertRaises(TypeError, f.unbind) + f.unbind(event) + self.assertEqual(f.bind(event), '') + self.assertEqual(f.bind(), ()) + + def test_unbind2(self): + f = self.frame + event = '' + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + + funcid = f.bind(event, test1) + funcid2 = f.bind(event, test2, add=True) + + f.unbind(event, funcid) + script = f.bind(event) + self.assertNotIn(funcid, script) + self.assertCommandNotExist(funcid) + self.assertCommandExist(funcid2) + + f.unbind(event, funcid2) + self.assertEqual(f.bind(event), '') + self.assertEqual(f.bind(), ()) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) + + # non-idempotent + self.assertRaises(tkinter.TclError, f.unbind, event, funcid2) + + def test_bind_rebind(self): + event = '' + f = self.frame + self.assertEqual(f.bind(), ()) + self.assertEqual(f.bind(event), '') + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = f.bind(event, test1) + funcid2 = f.bind(event, test2, add=True) + script = f.bind(event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = f.bind(event, test3) + script = f.bind(event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid3) + + def test_bind_class(self): + event = '' + bind_class = self.root.bind_class + unbind_class = self.root.unbind_class + self.assertRaises(TypeError, bind_class) + self.assertEqual(bind_class('Test'), ()) + self.assertEqual(bind_class('Test', event), '') + self.addCleanup(unbind_class, 'Test', event) + def test1(e): pass + def test2(e): pass + + funcid = bind_class('Test', event, test1) + self.assertEqual(bind_class('Test'), (event,)) + script = bind_class('Test', event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = bind_class('Test', event, test2, add=True) + script = bind_class('Test', event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def test_unbind_class(self): + event = '' + bind_class = self.root.bind_class + unbind_class = self.root.unbind_class + self.assertEqual(bind_class('Test'), ()) + self.assertEqual(bind_class('Test', event), '') + self.addCleanup(unbind_class, 'Test', event) + def test1(e): pass + def test2(e): pass + + funcid = bind_class('Test', event, test1) + funcid2 = bind_class('Test', event, test2, add=True) + + self.assertRaises(TypeError, unbind_class) + self.assertRaises(TypeError, unbind_class, 'Test') + unbind_class('Test', event) + self.assertEqual(bind_class('Test', event), '') + self.assertEqual(bind_class('Test'), ()) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + unbind_class('Test', event) # idempotent + + def test_bind_class_rebind(self): + event = '' + bind_class = self.root.bind_class + unbind_class = self.root.unbind_class + self.assertEqual(bind_class('Test'), ()) + self.assertEqual(bind_class('Test', event), '') + self.addCleanup(unbind_class, 'Test', event) + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = bind_class('Test', event, test1) + funcid2 = bind_class('Test', event, test2, add=True) + script = bind_class('Test', event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = bind_class('Test', event, test3) + script = bind_class('Test', event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + self.assertCommandExist(funcid3) + + def test_bind_all(self): + event = '' + bind_all = self.root.bind_all + unbind_all = self.root.unbind_all + self.assertNotIn(event, bind_all()) + self.assertEqual(bind_all(event), '') + self.addCleanup(unbind_all, event) + def test1(e): pass + def test2(e): pass + + funcid = bind_all(event, test1) + self.assertIn(event, bind_all()) + script = bind_all(event) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + + funcid2 = bind_all(event, test2, add=True) + script = bind_all(event) + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + def test_unbind_all(self): + event = '' + bind_all = self.root.bind_all + unbind_all = self.root.unbind_all + self.assertNotIn(event, bind_all()) + self.assertEqual(bind_all(event), '') + self.addCleanup(unbind_all, event) + def test1(e): pass + def test2(e): pass + + funcid = bind_all(event, test1) + funcid2 = bind_all(event, test2, add=True) + + unbind_all(event) + self.assertEqual(bind_all(event), '') + self.assertNotIn(event, bind_all()) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + unbind_all(event) # idempotent + + def test_bind_all_rebind(self): + event = '' + bind_all = self.root.bind_all + unbind_all = self.root.unbind_all + self.assertNotIn(event, bind_all()) + self.assertEqual(bind_all(event), '') + self.addCleanup(unbind_all, event) + def test1(e): pass + def test2(e): pass + def test3(e): pass + + funcid = bind_all(event, test1) + funcid2 = bind_all(event, test2, add=True) + script = bind_all(event) + self.assertIn(funcid2, script) + self.assertIn(funcid, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + + funcid3 = bind_all(event, test3) + script = bind_all(event) + self.assertNotIn(funcid, script) + self.assertNotIn(funcid2, script) + self.assertIn(funcid3, script) + self.assertCommandExist(funcid) + self.assertCommandExist(funcid2) + self.assertCommandExist(funcid3) + + def test_bindtags(self): + f = self.frame + self.assertEqual(self.root.bindtags(), ('.', 'Tk', 'all')) + self.assertEqual(f.bindtags(), (str(f), 'Test', '.', 'all')) + f.bindtags(('a', 'b c')) + self.assertEqual(f.bindtags(), ('a', 'b c')) + + def test_bind_events(self): + event = '' + root = self.root + t = tkinter.Toplevel(root) + f = tkinter.Frame(t, class_='Test', width=150, height=100) + f.pack() + root.wait_visibility() # needed on Windows + root.update_idletasks() + self.addCleanup(root.unbind_class, 'Test', event) + self.addCleanup(root.unbind_class, 'Toplevel', event) + self.addCleanup(root.unbind_class, 'tag', event) + self.addCleanup(root.unbind_class, 'tag2', event) + self.addCleanup(root.unbind_all, event) + def test(what): + return lambda e: events.append((what, e.widget)) + + root.bind_all(event, test('all')) + root.bind_class('Test', event, test('frame class')) + root.bind_class('Toplevel', event, test('toplevel class')) + root.bind_class('tag', event, test('tag')) + root.bind_class('tag2', event, test('tag2')) + f.bind(event, test('frame')) + t.bind(event, test('toplevel')) + + events = [] + f.event_generate(event) + self.assertEqual(events, [ + ('frame', f), + ('frame class', f), + ('toplevel', f), + ('all', f), + ]) + + events = [] + t.event_generate(event) + self.assertEqual(events, [ + ('toplevel', t), + ('toplevel class', t), + ('all', t), + ]) + + f.bindtags(('tag', 'tag3')) + events = [] + f.event_generate(event) + self.assertEqual(events, [('tag', f)]) + + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): def test_default_root(self): @@ -480,5 +783,9 @@ def test_mainloop(self): self.assertRaises(RuntimeError, tkinter.mainloop) +def _info_commands(widget, pattern=None): + return widget.tk.splitlist(widget.tk.call('info', 'commands', pattern)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index 328e4256ce0711..f809c4510e3a1f 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -10,6 +10,7 @@ class TextTest(AbstractTkTest, unittest.TestCase): def setUp(self): super().setUp() self.text = tkinter.Text(self.root) + self.text.pack() def test_debug(self): text = self.text @@ -41,8 +42,6 @@ def test_search(self): self.assertEqual(text.search('test', '1.0', 'end'), '1.3') def test_count(self): - # XXX Some assertions do not check against the intended result, - # but instead check the current result to prevent regression. text = self.text text.insert('1.0', 'Lorem ipsum dolor sit amet,\n' @@ -53,44 +52,27 @@ def test_count(self): options = ('chars', 'indices', 'lines', 'displaychars', 'displayindices', 'displaylines', 'xpixels', 'ypixels') - if self.wantobjects: - self.assertEqual(len(text.count('1.0', 'end', *options)), 8) - else: - text.count('1.0', 'end', *options) - self.assertEqual(text.count('1.0', 'end', 'chars', 'lines'), (124, 4) - if self.wantobjects else '124 4') - self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3) - if self.wantobjects else '92 3') - self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3) - if self.wantobjects else '-92 -3') - self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0) - if self.wantobjects else '0 0') - self.assertEqual(text.count('1.0', 'end', 'lines'), (4,) - if self.wantobjects else ('4',)) - self.assertEqual(text.count('end', '1.0', 'lines'), (-4,) - if self.wantobjects else ('-4',)) - self.assertEqual(text.count('1.3', '1.5', 'lines'), None - if self.wantobjects else ('0',)) - self.assertEqual(text.count('1.3', '1.3', 'lines'), None - if self.wantobjects else ('0',)) - self.assertEqual(text.count('1.0', 'end'), (124,) # 'indices' by default - if self.wantobjects else ('124',)) + self.assertEqual(len(text.count('1.0', 'end', *options)), 8) + self.assertEqual(text.count('1.0', 'end', 'chars', 'lines'), (124, 4)) + self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3)) + self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3)) + self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0)) + self.assertEqual(text.count('1.0', 'end', 'lines'), 4) + self.assertEqual(text.count('end', '1.0', 'lines'), -4) + self.assertEqual(text.count('1.3', '1.5', 'lines'), 0) + self.assertEqual(text.count('1.3', '1.3', 'lines'), 0) + self.assertEqual(text.count('1.0', 'end'), 124) # 'indices' by default + self.assertEqual(text.count('1.0', 'end', 'indices'), 124) self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') - self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) - self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int - if self.wantobjects else str) - self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), None - if self.wantobjects else '0') - self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2 - if self.wantobjects else '2') - self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), None - if self.wantobjects else '0') - self.assertEqual(text.count('1.3', '1.5', 'update'), (2,) - if self.wantobjects else ('2',)) - self.assertEqual(text.count('1.3', '1.3', 'update'), None - if self.wantobjects else ('0',)) + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), int) + self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), 0) + self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2) + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), 0) + self.assertEqual(text.count('1.3', '1.5', 'update'), 2) + self.assertEqual(text.count('1.3', '1.3', 'update'), 0) if __name__ == "__main__": diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index a0f7ad81bd540d..0c5d7c9c8c50d3 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -313,6 +313,8 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) expected = [b'Traceback (most recent call last):', b' File "", line 8, in __init__', + b' x = 1 / 0', + b' ^^^^^', b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) @@ -922,8 +924,63 @@ def f(): f" File \"{__file__}\", line {self.callable_line}, in get_exception", " callable()", f" File \"{__file__}\", line {f.__code__.co_firstlineno + 4}, in f", - " print(1, ｗｗｗ(", - " ^^^^", + f" print(1, ｗｗｗ(", + f" ^^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_with_wide_characters_term_highlight(self): + def f(): + 说明说明 = 1 + şçöğıĤellö = 0 # not wide but still non-ascii + return 说明说明 / şçöğıĤellö + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 3}, in f", + f" return 说明说明 / şçöğıĤellö", + f" ~~~~~~~~~^~~~~~~~~~~~", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_with_emojis_term_highlight(self): + def f(): + return "✨🐍" + func_说明说明("📗🚛", + "📗🚛") + "🐍" + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + f' return "✨🐍" + func_说明说明("📗🚛",', + f" ^^^^^^^^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_wide_chars_subscript(self): + def f(): + my_dct = { + "✨🚛✨": { + "说明": { + "🐍🐍🐍": None + } + } + } + return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"] + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 8}, in f", + f' return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"]', + f" ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^", ] self.assertEqual(actual, expected) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1a1e0a259fd042..7f60bf4bbc1e75 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2011,13 +2011,13 @@ def test_callable_instance_type_error(self): def f(): pass with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], None]) + isinstance(f, Callable[[], None]) with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], Any]) + isinstance(f, Callable[[], Any]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], None]) + isinstance(None, Callable[[], None]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], Any]) + isinstance(None, Callable[[], Any]) def test_repr(self): Callable = self.Callable diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 83237f5fe0d1b3..2c523230e7e97f 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1233,6 +1233,10 @@ def test_conflicting_envvar_and_command_line(self): self.assertEqual(stderr.splitlines(), [b"Traceback (most recent call last):", b" File \"\", line 1, in ", + b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' + b"arn('Message', DeprecationWarning)", + b' ^^^^^^^^^^' + b'^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', b"DeprecationWarning: Message"]) def test_default_filter_configuration(self): diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 519d2ba1a231b5..fb6b0b3e23d75e 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2246,6 +2246,22 @@ def test_seek_tell(self): fp.seek(0, os.SEEK_SET) self.assertEqual(fp.tell(), 0) + def test_read_after_seek(self): + # Issue 102956: Make sure seek(x, os.SEEK_CUR) doesn't break read() + txt = b"Charge men!" + bloc = txt.find(b"men") + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.read(6) + fp.seek(1, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + @requires_bz2() def test_decompress_without_3rd_party_library(self): data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py index c3ea567103275d..4b16ecc31156a5 100644 --- a/Lib/test/test_zoneinfo/__init__.py +++ b/Lib/test/test_zoneinfo/__init__.py @@ -1,2 +1,5 @@ -from .test_zoneinfo import * -from .test_zoneinfo_property import * +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 440e7f100c8c47..47b93d31fd3ffd 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -3715,25 +3715,28 @@ def compare(self, index1, op, index2): return self.tk.getboolean(self.tk.call( self._w, 'compare', index1, op, index2)) - def count(self, index1, index2, *args): # new in Tk 8.5 + def count(self, index1, index2, *options): # new in Tk 8.5 """Counts the number of relevant things between the two indices. - If index1 is after index2, the result will be a negative number + + If INDEX1 is after INDEX2, the result will be a negative number (and this holds for each of the possible options). - The actual items which are counted depends on the options given by - args. The result is a list of integers, one for the result of each - counting option given. Valid counting options are "chars", + The actual items which are counted depends on the options given. + The result is a tuple of integers, one for the result of each + counting option given, if more than one option is specified, + otherwise it is an integer. Valid counting options are "chars", "displaychars", "displayindices", "displaylines", "indices", - "lines", "xpixels" and "ypixels". There is an additional possible + "lines", "xpixels" and "ypixels". The default value, if no + option is specified, is "indices". There is an additional possible option "update", which if given then all subsequent options ensure that any possible out of date information is recalculated.""" - args = ['-%s' % arg for arg in args] - args += [index1, index2] - res = self.tk.call(self._w, 'count', *args) or None - if res is not None and len(args) <= 3: - return (res, ) - else: - return res + options = ['-%s' % arg for arg in options] + res = self.tk.call(self._w, 'count', *options, index1, index2) + if not isinstance(res, int): + res = self._getints(res) + if len(res) == 1: + res, = res + return res def debug(self, boolean=None): """Turn on the internal consistency checks of the B-Tree inside the text diff --git a/Lib/traceback.py b/Lib/traceback.py index 7cc84b9c762aeb..0d41c3432eda2c 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -153,14 +153,11 @@ def format_exception_only(exc, /, value=_sentinel): The return value is a list of strings, each ending in a newline. - Normally, the list contains a single string; however, for - SyntaxError exceptions, it contains several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the list. - + The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's ``__notes__``. """ if value is _sentinel: value = exc @@ -479,17 +476,17 @@ def format_frame_summary(self, frame_summary): gets called for every frame to be printed in the stack summary. """ row = [] - if frame_summary.filename.startswith("", line {}, in {}\n'.format( - frame_summary.lineno, frame_summary.name)) - else: - row.append(' File "{}", line {}, in {}\n'.format( - frame_summary.filename, frame_summary.lineno, frame_summary.name)) + filename = frame_summary.filename + if frame_summary.filename.startswith("-"): + filename = "" + row.append(' File "{}", line {}, in {}\n'.format( + filename, frame_summary.lineno, frame_summary.name)) if frame_summary.line: stripped_line = frame_summary.line.strip() row.append(' {}\n'.format(stripped_line)) - orig_line_len = len(frame_summary._original_line) + line = frame_summary._original_line + orig_line_len = len(line) frame_line_len = len(frame_summary.line.lstrip()) stripped_characters = orig_line_len - frame_line_len if ( @@ -497,31 +494,40 @@ def format_frame_summary(self, frame_summary): and frame_summary.end_colno is not None ): start_offset = _byte_offset_to_character_offset( - frame_summary._original_line, frame_summary.colno) + 1 + line, frame_summary.colno) end_offset = _byte_offset_to_character_offset( - frame_summary._original_line, frame_summary.end_colno) + 1 + line, frame_summary.end_colno) + code_segment = line[start_offset:end_offset] anchors = None if frame_summary.lineno == frame_summary.end_lineno: with suppress(Exception): - anchors = _extract_caret_anchors_from_line_segment( - frame_summary._original_line[start_offset - 1:end_offset - 1] - ) + anchors = _extract_caret_anchors_from_line_segment(code_segment) else: - end_offset = stripped_characters + len(stripped_line) + # Don't count the newline since the anchors only need to + # go up until the last character of the line. + end_offset = len(line.rstrip()) # show indicators if primary char doesn't span the frame line if end_offset - start_offset < len(stripped_line) or ( anchors and anchors.right_start_offset - anchors.left_end_offset > 0): + # When showing this on a terminal, some of the non-ASCII characters + # might be rendered as double-width characters, so we need to take + # that into account when calculating the length of the line. + dp_start_offset = _display_width(line, start_offset) + 1 + dp_end_offset = _display_width(line, end_offset) + 1 + row.append(' ') - row.append(' ' * (start_offset - stripped_characters)) + row.append(' ' * (dp_start_offset - stripped_characters)) if anchors: - row.append(anchors.primary_char * (anchors.left_end_offset)) - row.append(anchors.secondary_char * (anchors.right_start_offset - anchors.left_end_offset)) - row.append(anchors.primary_char * (end_offset - start_offset - anchors.right_start_offset)) + dp_left_end_offset = _display_width(code_segment, anchors.left_end_offset) + dp_right_start_offset = _display_width(code_segment, anchors.right_start_offset) + row.append(anchors.primary_char * dp_left_end_offset) + row.append(anchors.secondary_char * (dp_right_start_offset - dp_left_end_offset)) + row.append(anchors.primary_char * (dp_end_offset - dp_start_offset - dp_right_start_offset)) else: - row.append('^' * (end_offset - start_offset)) + row.append('^' * (dp_end_offset - dp_start_offset)) row.append('\n') @@ -642,6 +648,25 @@ def _extract_caret_anchors_from_line_segment(segment): return None +_WIDE_CHAR_SPECIFIERS = "WF" + +def _display_width(line, offset): + """Calculate the extra amount of width space the given source + code segment might take if it were to be displayed on a fixed + width output device. Supports wide unicode characters and emojis.""" + + # Fast track for ASCII-only strings + if line.isascii(): + return offset + + import unicodedata + + return sum( + 2 if unicodedata.east_asian_width(char) in _WIDE_CHAR_SPECIFIERS else 1 + for char in line[:offset] + ) + + class _ExceptionPrintContext: def __init__(self): @@ -857,13 +882,13 @@ def format_exception_only(self, *, show_group=False, _depth=0): The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. + Generator yields the exception message. + For :exc:`SyntaxError` exceptions, it + also yields (before the exception message) + several lines that (when printed) + display detailed information about where the syntax error occurred. + Following the message, generator also yields + all the exception's ``__notes__``. """ indent = 3 * _depth * ' ' diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index f6c9d6aa6f9a32..2ab6c15e2c079e 100755 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -161,7 +161,7 @@ def __init__(self, filename=None): label='Help', underline=0) root['menu'] = self.mBar - pane = PanedWindow(orient=HORIZONTAL, sashwidth=5, + pane = PanedWindow(root, orient=HORIZONTAL, sashwidth=5, sashrelief=SOLID, bg='#ddd') pane.add(self.makeTextFrame(pane)) pane.add(self.makeGraphFrame(pane)) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 995d2678e87536..2b28a079dbaa95 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1136,8 +1136,12 @@ def seek(self, offset, whence=os.SEEK_SET): read_offset = new_pos - curr_pos buff_offset = read_offset + self._offset + if buff_offset >= 0 and buff_offset < len(self._readbuffer): + # Just move the _offset index if the new position is in the _readbuffer + self._offset = buff_offset + read_offset = 0 # Fast seek uncompressed unencrypted file - if self._compress_type == ZIP_STORED and self._decrypter is None and read_offset > 0: + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset > 0: # disable CRC checking after first seeking - it would be invalid self._expected_crc = None # seek actual file taking already buffered data into account @@ -1148,10 +1152,6 @@ def seek(self, offset, whence=os.SEEK_SET): # flush read buffer self._readbuffer = b'' self._offset = 0 - elif buff_offset >= 0 and buff_offset < len(self._readbuffer): - # Just move the _offset index if the new position is in the _readbuffer - self._offset = buff_offset - read_offset = 0 elif read_offset < 0: # Position is before the current position. Reset the ZipExtFile self._fileobj.seek(self._orig_compress_start) diff --git a/Misc/ACKS b/Misc/ACKS index 94cb1965676f48..812aa1be6e796a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1345,6 +1345,7 @@ Michele Orrù Tomáš Orsava Oleg Oshmyan Denis Osipov +Savannah Ostrowski Denis S. Otkidach Peter Otten Michael Otteneder diff --git a/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst b/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst index 49b5da32f8f72e..399054040f2ec7 100644 --- a/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst +++ b/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst @@ -1,4 +1,5 @@ The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, -``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions are -now built with the :ref:`limited C API `. +``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, +``_testimportmultiple`` and ``_uuid`` C extensions are now built with the +:ref:`limited C API `. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2023-10-17-03-10-40.gh-issue-110828.31vQ9B.rst b/Misc/NEWS.d/next/Build/2023-10-17-03-10-40.gh-issue-110828.31vQ9B.rst new file mode 100644 index 00000000000000..13647fee058021 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-17-03-10-40.gh-issue-110828.31vQ9B.rst @@ -0,0 +1 @@ +AIX 32bit needs ``-latomic`` to build the :mod:`!_testcapi` extension module. diff --git a/Misc/NEWS.d/next/Build/2023-10-20-15-29-31.gh-issue-111046.2DxQl8.rst b/Misc/NEWS.d/next/Build/2023-10-20-15-29-31.gh-issue-111046.2DxQl8.rst new file mode 100644 index 00000000000000..446b8b612862f9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-20-15-29-31.gh-issue-111046.2DxQl8.rst @@ -0,0 +1 @@ +For wasi-threads, memory is now exported to fix compatibility issues with some wasm runtimes. diff --git a/Misc/NEWS.d/next/C API/2023-06-08-21-12-44.gh-issue-67565.UkK3x-.rst b/Misc/NEWS.d/next/C API/2023-06-08-21-12-44.gh-issue-67565.UkK3x-.rst new file mode 100644 index 00000000000000..0e50976b0e1ddf --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-06-08-21-12-44.gh-issue-67565.UkK3x-.rst @@ -0,0 +1 @@ +Remove redundant C-contiguity check in :file:`getargs.c`, :mod:`binascii`, :mod:`ssl` and Argument Clinic. Patched by Stefan Krah and Furkan Onder diff --git a/Misc/NEWS.d/next/C API/2023-10-20-01-42-43.gh-issue-111089.VIrd5q.rst b/Misc/NEWS.d/next/C API/2023-10-20-01-42-43.gh-issue-111089.VIrd5q.rst new file mode 100644 index 00000000000000..2008dd5438d2b5 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-10-20-01-42-43.gh-issue-111089.VIrd5q.rst @@ -0,0 +1,2 @@ +The :c:func:`PyUnicode_AsUTF8` function now raises an exception if the +string contains embedded null characters. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2023-10-20-18-07-24.gh-issue-111089.RxkyrQ.rst b/Misc/NEWS.d/next/C API/2023-10-20-18-07-24.gh-issue-111089.RxkyrQ.rst new file mode 100644 index 00000000000000..fe32e06fe4f063 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-10-20-18-07-24.gh-issue-111089.RxkyrQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnicode_AsUTF8` function to the limited C API. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst new file mode 100644 index 00000000000000..332afb61e349c9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst @@ -0,0 +1,3 @@ +Replace ``prev_instr`` on the interpreter frame by ``instr_ptr`` which +points to the beginning of the instruction that is currently executing (or +will execute once the frame resumes). diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-15-22-18-45.gh-issue-109894.UAmo06.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-15-22-18-45.gh-issue-109894.UAmo06.rst new file mode 100644 index 00000000000000..214853660c5a57 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-15-22-18-45.gh-issue-109894.UAmo06.rst @@ -0,0 +1 @@ +Fixed crash due to improperly initialized static :exc:`MemoryError` in subinterpreter. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-16-15-51-37.gh-issue-109214.-RGTFH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-16-15-51-37.gh-issue-109214.-RGTFH.rst new file mode 100644 index 00000000000000..c24f18cee71fa0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-16-15-51-37.gh-issue-109214.-RGTFH.rst @@ -0,0 +1 @@ +Remove unnecessary instruction pointer updates before returning from frames. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-20-23-14-06.gh-issue-111123.jjVc3M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-20-23-14-06.gh-issue-111123.jjVc3M.rst new file mode 100644 index 00000000000000..f2cebe287db3ee --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-20-23-14-06.gh-issue-111123.jjVc3M.rst @@ -0,0 +1,2 @@ +Fix a bug where a :keyword:`global` declaration in an :keyword:`except` block +is rejected when the global is used in the :keyword:`else` block. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst new file mode 100644 index 00000000000000..b0474f38cff6f6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst @@ -0,0 +1,2 @@ +Show source lines in tracebacks when using the ``-c`` option when running +Python. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-23-22-11-09.gh-issue-94438.y2pITu.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-23-22-11-09.gh-issue-94438.y2pITu.rst new file mode 100644 index 00000000000000..b6e147a48a8cd8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-23-22-11-09.gh-issue-94438.y2pITu.rst @@ -0,0 +1 @@ +Fix a regression that prevented jumping across ``is None`` and ``is not None`` when debugging. Patch by Savannah Ostrowski. diff --git a/Misc/NEWS.d/next/Library/2022-10-14-21-11-10.gh-issue-97928.Pdxh1G.rst b/Misc/NEWS.d/next/Library/2022-10-14-21-11-10.gh-issue-97928.Pdxh1G.rst new file mode 100644 index 00000000000000..4acf396f840d61 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-14-21-11-10.gh-issue-97928.Pdxh1G.rst @@ -0,0 +1,6 @@ +Change the behavior of :meth:`tkinter.Text.count`. It now always returns an +integer if one or less counting options are specified. Previously it could +return a single count as a 1-tuple, an integer (only if option ``"update"`` +was specified) or ``None`` if no items found. The result is now the same if +``wantobjects`` is set to ``0``. + diff --git a/Misc/NEWS.d/next/Library/2023-04-15-14-45-21.gh-issue-102956.Z6qeUy.rst b/Misc/NEWS.d/next/Library/2023-04-15-14-45-21.gh-issue-102956.Z6qeUy.rst new file mode 100644 index 00000000000000..1a4bb9bc0dc46a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-15-14-45-21.gh-issue-102956.Z6qeUy.rst @@ -0,0 +1 @@ +Fix returning of empty byte strings after seek in zipfile module diff --git a/Misc/NEWS.d/next/Library/2023-08-30-19-10-35.gh-issue-105931.Lpwve8.rst b/Misc/NEWS.d/next/Library/2023-08-30-19-10-35.gh-issue-105931.Lpwve8.rst new file mode 100644 index 00000000000000..4e769ec4c47561 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-30-19-10-35.gh-issue-105931.Lpwve8.rst @@ -0,0 +1,8 @@ +Change :mod:`compileall` to only strip the stripdir prefix from the full path +recorded in the compiled ``.pyc`` file, when the prefix matches the start of +the full path in its entirety. When the prefix does not match, no stripping is +performed and a warning to this effect is displayed. + +Previously all path components of the stripdir prefix that matched the full +path were removed, while those that did not match were left alone (including +ones interspersed between matching components). diff --git a/Misc/NEWS.d/next/Library/2023-10-02-05-23-27.gh-issue-110196.djwt0z.rst b/Misc/NEWS.d/next/Library/2023-10-02-05-23-27.gh-issue-110196.djwt0z.rst new file mode 100644 index 00000000000000..341f3380fffd60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-02-05-23-27.gh-issue-110196.djwt0z.rst @@ -0,0 +1 @@ +Add ``__reduce__`` method to :class:`IPv6Address` in order to keep ``scope_id`` diff --git a/Misc/NEWS.d/next/Library/2023-10-19-22-46-34.gh-issue-111092.hgut12.rst b/Misc/NEWS.d/next/Library/2023-10-19-22-46-34.gh-issue-111092.hgut12.rst new file mode 100644 index 00000000000000..487bd177d27e31 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-19-22-46-34.gh-issue-111092.hgut12.rst @@ -0,0 +1 @@ +Make turtledemo run without default root enabled. diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-29-10.gh-issue-110910.u2oPwX.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-29-10.gh-issue-110910.u2oPwX.rst new file mode 100644 index 00000000000000..c750447e9fe4a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-20-15-29-10.gh-issue-110910.u2oPwX.rst @@ -0,0 +1,3 @@ +Fix invalid state handling in :class:`asyncio.TaskGroup` and +:class:`asyncio.Timeout`. They now raise proper RuntimeError if they are +improperly used and are left in consistent state after this. diff --git a/Misc/NEWS.d/next/Library/2023-10-21-13-57-06.gh-issue-111159.GoHp7s.rst b/Misc/NEWS.d/next/Library/2023-10-21-13-57-06.gh-issue-111159.GoHp7s.rst new file mode 100644 index 00000000000000..bdec4f4443d80b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-21-13-57-06.gh-issue-111159.GoHp7s.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` output comparison for exceptions with notes. diff --git a/Misc/NEWS.d/next/Library/2023-10-22-21-28-05.gh-issue-111187._W11Ab.rst b/Misc/NEWS.d/next/Library/2023-10-22-21-28-05.gh-issue-111187._W11Ab.rst new file mode 100644 index 00000000000000..dc2424370bb96c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-22-21-28-05.gh-issue-111187._W11Ab.rst @@ -0,0 +1 @@ +Postpone removal version for locale.getdefaultlocale() to Python 3.15. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-13-53-58.gh-issue-111174.Oohmzd.rst b/Misc/NEWS.d/next/Library/2023-10-23-13-53-58.gh-issue-111174.Oohmzd.rst new file mode 100644 index 00000000000000..95c315404d0ee6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-23-13-53-58.gh-issue-111174.Oohmzd.rst @@ -0,0 +1,2 @@ +Fix crash in :meth:`io.BytesIO.getbuffer` called repeatedly for empty +BytesIO. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-22-40-47.gh-issue-111230.k3Jm84.rst b/Misc/NEWS.d/next/Library/2023-10-23-22-40-47.gh-issue-111230.k3Jm84.rst new file mode 100644 index 00000000000000..61d035853db070 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-23-22-40-47.gh-issue-111230.k3Jm84.rst @@ -0,0 +1 @@ +Fix :mod:`ssl` not checking for errors when initializing. diff --git a/Misc/NEWS.d/next/Library/2023-10-23-23-14-54.gh-issue-111233.sCdCC0.rst b/Misc/NEWS.d/next/Library/2023-10-23-23-14-54.gh-issue-111233.sCdCC0.rst new file mode 100644 index 00000000000000..86d622a74095ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-23-23-14-54.gh-issue-111233.sCdCC0.rst @@ -0,0 +1 @@ +Fix :mod:`select` not checking for errors when initializing. diff --git a/Misc/NEWS.d/next/Library/2023-10-24-12-09-46.gh-issue-111251.urFYtn.rst b/Misc/NEWS.d/next/Library/2023-10-24-12-09-46.gh-issue-111251.urFYtn.rst new file mode 100644 index 00000000000000..3a87cb25da5cb4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-24-12-09-46.gh-issue-111251.urFYtn.rst @@ -0,0 +1 @@ +Fix :mod:`_blake2` not checking for errors when initializing. diff --git a/Misc/NEWS.d/next/Library/2023-10-24-12-20-46.gh-issue-111253.HFywSK.rst b/Misc/NEWS.d/next/Library/2023-10-24-12-20-46.gh-issue-111253.HFywSK.rst new file mode 100644 index 00000000000000..e21a42605aeaf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-24-12-20-46.gh-issue-111253.HFywSK.rst @@ -0,0 +1 @@ +Add error checking during :mod:`!_socket` module init. diff --git a/Misc/NEWS.d/next/Library/2023-10-25-08-42-05.gh-issue-111295.H2K4lf.rst b/Misc/NEWS.d/next/Library/2023-10-25-08-42-05.gh-issue-111295.H2K4lf.rst new file mode 100644 index 00000000000000..28b85ec3eadab7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-25-08-42-05.gh-issue-111295.H2K4lf.rst @@ -0,0 +1 @@ +Fix :mod:`time` not checking for errors when initializing. diff --git a/Misc/NEWS.d/next/Library/2023-10-25-11-13-35.gh-issue-111259.z7ndeA.rst b/Misc/NEWS.d/next/Library/2023-10-25-11-13-35.gh-issue-111259.z7ndeA.rst new file mode 100644 index 00000000000000..4b597f51a955a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-25-11-13-35.gh-issue-111259.z7ndeA.rst @@ -0,0 +1 @@ +Optimize recursive wildcards in :mod:`pathlib`. diff --git a/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst b/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst new file mode 100644 index 00000000000000..7cb79c0cbf29f1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst @@ -0,0 +1,4 @@ +Test case matching patterns specified by options ``--match``, ``--ignore``, +``--matchfile`` and ``--ignorefile`` are now tested in the order of +specification, and the last match determines whether the test case be run or +ignored. diff --git a/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst b/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst new file mode 100644 index 00000000000000..45bb0774a9abe3 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst @@ -0,0 +1,2 @@ +Fix regrtest if the ``SOURCE_DATE_EPOCH`` environment variable is defined: +use the variable value as the random seed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-10-21-19-27-36.gh-issue-111165.FU6mUk.rst b/Misc/NEWS.d/next/Tests/2023-10-21-19-27-36.gh-issue-111165.FU6mUk.rst new file mode 100644 index 00000000000000..11f302d943c29a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-21-19-27-36.gh-issue-111165.FU6mUk.rst @@ -0,0 +1,2 @@ +Remove no longer used functions ``run_unittest()`` and ``run_doctest()`` +from the :mod:`test.support` module. diff --git a/Misc/NEWS.d/next/Windows/2023-10-19-21-46-18.gh-issue-110913.CWlPfg.rst b/Misc/NEWS.d/next/Windows/2023-10-19-21-46-18.gh-issue-110913.CWlPfg.rst new file mode 100644 index 00000000000000..d4c1b56d98ef0e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-10-19-21-46-18.gh-issue-110913.CWlPfg.rst @@ -0,0 +1 @@ +WindowsConsoleIO now correctly chunks large buffers without splitting up UTF-8 sequences. diff --git a/Misc/NEWS.d/next/Windows/2023-10-25-05-01-28.gh-issue-111293.FSsLT6.rst b/Misc/NEWS.d/next/Windows/2023-10-25-05-01-28.gh-issue-111293.FSsLT6.rst new file mode 100644 index 00000000000000..4c6b255bc44c7b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-10-25-05-01-28.gh-issue-111293.FSsLT6.rst @@ -0,0 +1 @@ +Fix :data:`os.DirEntry.inode` dropping higher 64 bits of a file id on some filesystems on Windows. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 75c260c8f1b2be..0601de20fe0f46 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2478,3 +2478,5 @@ added = '3.13' [function.PySys_AuditTuple] added = '3.13' +[function.PyUnicode_AsUTF8] + added = '3.13' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 647f44280b9ea1..c73522b8ecf426 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -159,7 +159,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 0d1d88c6603684..5df9fd3df493ee 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -74,6 +74,12 @@ _blake2_free(void *module) Py_DECREF(x); \ } while(0) +#define ADD_INT_CONST(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + static int blake2_exec(PyObject *m) { @@ -95,10 +101,10 @@ blake2_exec(PyObject *m) ADD_INT(d, "MAX_KEY_SIZE", BLAKE2B_KEYBYTES); ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_SALT_SIZE", BLAKE2B_SALTBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_PERSON_SIZE", BLAKE2B_PERSONALBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_MAX_KEY_SIZE", BLAKE2B_KEYBYTES); - PyModule_AddIntConstant(m, "BLAKE2B_MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); + ADD_INT_CONST("BLAKE2B_SALT_SIZE", BLAKE2B_SALTBYTES); + ADD_INT_CONST("BLAKE2B_PERSON_SIZE", BLAKE2B_PERSONALBYTES); + ADD_INT_CONST("BLAKE2B_MAX_KEY_SIZE", BLAKE2B_KEYBYTES); + ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); /* BLAKE2s */ st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( @@ -117,14 +123,17 @@ blake2_exec(PyObject *m) ADD_INT(d, "MAX_KEY_SIZE", BLAKE2S_KEYBYTES); ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_SALT_SIZE", BLAKE2S_SALTBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_PERSON_SIZE", BLAKE2S_PERSONALBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_MAX_KEY_SIZE", BLAKE2S_KEYBYTES); - PyModule_AddIntConstant(m, "BLAKE2S_MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); + ADD_INT_CONST("BLAKE2S_SALT_SIZE", BLAKE2S_SALTBYTES); + ADD_INT_CONST("BLAKE2S_PERSON_SIZE", BLAKE2S_PERSONALBYTES); + ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", BLAKE2S_KEYBYTES); + ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); return 0; } +#undef ADD_INT +#undef ADD_INT_CONST + static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 79b6e672005503..47d62717eb76e7 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -98,10 +98,6 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("blake2b", "argument 'key'", "contiguous buffer", fastargs[2]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -110,10 +106,6 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("blake2b", "argument 'salt'", "contiguous buffer", fastargs[3]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -122,10 +114,6 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&person, 'C')) { - _PyArg_BadArgument("blake2b", "argument 'person'", "contiguous buffer", fastargs[4]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -277,4 +265,4 @@ _blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=19b82b55c033d895 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e18eeaee40623bfc input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index 20446b335f82ed..7a0f6eeff5b5b5 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -98,10 +98,6 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("blake2s", "argument 'key'", "contiguous buffer", fastargs[2]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -110,10 +106,6 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("blake2s", "argument 'salt'", "contiguous buffer", fastargs[3]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -122,10 +114,6 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&person, 'C')) { - _PyArg_BadArgument("blake2s", "argument 'person'", "contiguous buffer", fastargs[4]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -277,4 +265,4 @@ _blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=79b3479e90f4d077 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=24690e4e2586cafd input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index fc08c42bd3574a..2a0c26a0fec1b6 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1405,7 +1405,7 @@ static PyObject *load_library(PyObject *self, PyObject *args) #ifdef _WIN64 return PyLong_FromVoidPtr(hMod); #else - return Py_BuildValue("i", hMod); + return PyLong_FromLong((int)hMod); #endif } diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d339a8aa798361..d04d1e973af030 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3673,7 +3673,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) #endif /*[clinic input] -_curses.napms +_curses.napms -> int ms: int Duration in milliseconds. @@ -3682,13 +3682,13 @@ _curses.napms Sleep for specified time. [clinic start generated code]*/ -static PyObject * +static int _curses_napms_impl(PyObject *module, int ms) -/*[clinic end generated code: output=a40a1da2e39ea438 input=20cd3af2b6900f56]*/ +/*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { PyCursesInitialised; - return Py_BuildValue("i", napms(ms)); + return napms(ms); } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 684a628806fbb5..9938ed5f8095b5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4060,8 +4060,8 @@ static PyObject * timezone_getinitargs(PyDateTime_TimeZone *self, PyObject *Py_UNUSED(ignored)) { if (self->name == NULL) - return Py_BuildValue("(O)", self->offset); - return Py_BuildValue("(OO)", self->offset, self->name); + return PyTuple_Pack(1, self->offset); + return PyTuple_Pack(2, self->offset, self->name); } static PyMethodDef timezone_methods[] = { diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index f3074203f54ea2..16b8ac600ace79 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -126,12 +126,13 @@ unshare_buffer(bytesio *self, size_t size) static int resize_buffer(bytesio *self, size_t size) { + assert(self->buf != NULL); + assert(self->exports == 0); + /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ size_t alloc = PyBytes_GET_SIZE(self->buf); - assert(self->buf != NULL); - /* For simplicity, stay in the range of the signed type. Anyway, Python doesn't allow strings to be longer than this. */ if (size > PY_SSIZE_T_MAX) @@ -1074,7 +1075,7 @@ bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) "bytesiobuf_getbuffer: view==NULL argument is obsolete"); return -1; } - if (SHARED_BUF(b)) { + if (b->exports == 0 && SHARED_BUF(b)) { if (unshare_buffer(b, b->string_size) < 0) return -1; } diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 83c97ae42db00a..f6ac2699135917 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -30,10 +30,6 @@ _io__BufferedIOBase_readinto(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io__BufferedIOBase_readinto_impl(self, &buffer); exit: @@ -66,10 +62,6 @@ _io__BufferedIOBase_readinto1(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto1", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto1", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io__BufferedIOBase_readinto1_impl(self, &buffer); exit: @@ -592,10 +584,6 @@ _io__Buffered_readinto(buffered *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io__Buffered_readinto_impl(self, &buffer); exit: @@ -628,10 +616,6 @@ _io__Buffered_readinto1(buffered *self, PyObject *arg) _PyArg_BadArgument("readinto1", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto1", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io__Buffered_readinto1_impl(self, &buffer); exit: @@ -945,10 +929,6 @@ _io_BufferedWriter_write(buffered *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io_BufferedWriter_write_impl(self, &buffer); exit: @@ -1095,4 +1075,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b83f65fad0cd5fb6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=090e70253e35fc22 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index cf16335c935300..37023e49087647 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -332,10 +332,6 @@ _io_BytesIO_readinto(bytesio *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _io_BytesIO_readinto_impl(self, &buffer); exit: @@ -538,4 +534,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=27333725edff70a0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2be0e05a8871b7e2 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 748bc9c644fbfe..cf3ba28b066cf7 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -249,10 +249,6 @@ _io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io_FileIO_readinto_impl(self, cls, &buffer); exit: @@ -381,10 +377,6 @@ _io_FileIO_write(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssiz if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io_FileIO_write_impl(self, cls, &b); exit: @@ -536,4 +528,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=10838003d15e7b3d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1c0f4a36f76b0c6a input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index f40786a96caf72..6cab295c44611d 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -247,10 +247,6 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject * _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("readinto", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io__WindowsConsoleIO_readinto_impl(self, cls, &buffer); exit: @@ -391,10 +387,6 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyTypeObject *cls, PyObject *con if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = _io__WindowsConsoleIO_write_impl(self, cls, &b); exit: @@ -465,4 +457,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=2debef253fa1ab90 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=04108fc26b187386 input=a9049054013a1b77]*/ diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 50b8818aad410b..6680488b740cfc 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -134,6 +134,23 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { return m; } +static DWORD +_find_last_utf8_boundary(const char *buf, DWORD len) +{ + /* This function never returns 0, returns the original len instead */ + DWORD count = 1; + if (len == 0 || (buf[len - 1] & 0x80) == 0) { + return len; + } + for (;; count++) { + if (count > 3 || count >= len) { + return len; + } + if ((buf[len - count] & 0xc0) != 0x80) { + return len - count; + } + } +} /*[clinic input] module _io @@ -975,7 +992,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, { BOOL res = TRUE; wchar_t *wbuf; - DWORD len, wlen, orig_len, n = 0; + DWORD len, wlen, n = 0; HANDLE handle; if (self->fd == -1) @@ -1007,21 +1024,8 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, have to reduce and recalculate. */ while (wlen > 32766 / sizeof(wchar_t)) { len /= 2; - orig_len = len; - /* Reduce the length until we hit the final byte of a UTF-8 sequence - * (top bit is unset). Fix for github issue 82052. - */ - while (len > 0 && (((char *)b->buf)[len-1] & 0x80) != 0) - --len; - /* If we hit a length of 0, something has gone wrong. This shouldn't - * be possible, as valid UTF-8 can have at most 3 non-final bytes - * before a final one, and our buffer is way longer than that. - * But to be on the safe side, if we hit this issue we just restore - * the original length and let the console API sort it out. - */ - if (len == 0) { - len = orig_len; - } + /* Fix for github issues gh-110913 and gh-82052. */ + len = _find_last_utf8_boundary(b->buf, len); wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); } Py_END_ALLOW_THREADS diff --git a/Modules/_multiprocessing/clinic/multiprocessing.c.h b/Modules/_multiprocessing/clinic/multiprocessing.c.h index 70cdeef4821b33..6d4f5c2afcfd39 100644 --- a/Modules/_multiprocessing/clinic/multiprocessing.c.h +++ b/Modules/_multiprocessing/clinic/multiprocessing.c.h @@ -104,10 +104,6 @@ _multiprocessing_send(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &buf, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buf, 'C')) { - _PyArg_BadArgument("send", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _multiprocessing_send_impl(module, handle, &buf); exit: @@ -168,4 +164,4 @@ _multiprocessing_sem_unlink(PyObject *module, PyObject *arg) #ifndef _MULTIPROCESSING_SEND_METHODDEF #define _MULTIPROCESSING_SEND_METHODDEF #endif /* !defined(_MULTIPROCESSING_SEND_METHODDEF) */ -/*[clinic end generated code: output=48504f7a2d37958c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73b4cb8428d816da input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 6b6a9b11c0d990..1b894ea4c67adc 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - #if defined(HAVE_SHM_OPEN) PyDoc_STRVAR(_posixshmem_shm_open__doc__, @@ -17,69 +11,25 @@ PyDoc_STRVAR(_posixshmem_shm_open__doc__, "Open a shared memory object. Returns a file descriptor (integer)."); #define _POSIXSHMEM_SHM_OPEN_METHODDEF \ - {"shm_open", _PyCFunction_CAST(_posixshmem_shm_open), METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_open__doc__}, + {"shm_open", (PyCFunction)(void(*)(void))_posixshmem_shm_open, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_open__doc__}, static int _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, int mode); static PyObject * -_posixshmem_shm_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), &_Py_ID(flags), &_Py_ID(mode), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", "flags", "mode", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "shm_open", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + static char *_keywords[] = {"path", "flags", "mode", NULL}; PyObject *path; int flags; int mode = 511; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_open", "argument 'path'", "str", args[0]); - goto exit; - } - path = args[0]; - flags = PyLong_AsInt(args[1]); - if (flags == -1 && PyErr_Occurred()) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - mode = PyLong_AsInt(args[2]); - if (mode == -1 && PyErr_Occurred()) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ui|i:shm_open", _keywords, + &path, &flags, &mode)) goto exit; - } -skip_optional_pos: _return_value = _posixshmem_shm_open_impl(module, path, flags, mode); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -105,52 +55,21 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ - {"shm_unlink", _PyCFunction_CAST(_posixshmem_shm_unlink), METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, + {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * -_posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "shm_unlink", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"path", NULL}; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:shm_unlink", _keywords, + &path)) goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_unlink", "argument 'path'", "str", args[0]); - goto exit; - } - path = args[0]; return_value = _posixshmem_shm_unlink_impl(module, path); exit: @@ -166,4 +85,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=b525631f5915d7f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index f4da9f7ceba698..c4d1138534d8c5 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,18 +2,17 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ -// clinic/posixshmem.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif +// Need limited C API version 3.13 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +#define Py_LIMITED_API 0x030d0000 #include -// for shm_open() and shm_unlink() +#include // EINTR #ifdef HAVE_SYS_MMAN_H -#include +# include // shm_open(), shm_unlink() #endif + /*[clinic input] module _posixshmem [clinic start generated code]*/ diff --git a/Modules/_opcode.c b/Modules/_opcode.c index dac9c019b249e9..f0b547795b847a 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -244,8 +244,7 @@ _opcode_get_nb_ops_impl(PyObject *module) } #define ADD_NB_OP(NUM, STR) \ do { \ - PyObject *pair = Py_BuildValue( \ - "NN", PyUnicode_FromString(#NUM), PyUnicode_FromString(STR)); \ + PyObject *pair = Py_BuildValue("ss", #NUM, STR); \ if (pair == NULL) { \ Py_DECREF(list); \ return NULL; \ diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 7a220e2f3d5458..514bec16a347f4 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -71,7 +71,7 @@ #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_AsByteArray() +#include "pycore_long.h" // _PyLong_NumBits() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 2c638173222140..b95ba948aaf97f 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -88,10 +88,6 @@ blob_write(pysqlite_Blob *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = blob_write_impl(self, &data); exit: @@ -215,4 +211,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=27c24afc687bd772 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31abd55660e0c5af input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 46761cddd891e1..db5eb77891e52e 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -1557,10 +1557,6 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("deserialize", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (!noptargs) { goto skip_optional_kwonly; @@ -1822,4 +1818,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=b9c27a406e329587 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90b5b9c14261b8d7 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 319ed0c29c7a9b..ce46c1e69f3131 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -76,15 +76,10 @@ isolation_level_converter(PyObject *str_or_none, const char **result) *result = NULL; } else if (PyUnicode_Check(str_or_none)) { - Py_ssize_t sz; - const char *str = PyUnicode_AsUTF8AndSize(str_or_none, &sz); + const char *str = PyUnicode_AsUTF8(str_or_none); if (str == NULL) { return 0; } - if (strlen(str) != (size_t)sz) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return 0; - } const char *level = get_isolation_level(str); if (level == NULL) { diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 0f134b194de6f6..d451974b9cf81e 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -107,9 +107,11 @@ static unsigned int sre_toupper(unsigned int ch) { #if VERBOSE == 0 # define INIT_TRACE(state) +# define DO_TRACE 0 # define TRACE(v) #elif VERBOSE == 1 # define INIT_TRACE(state) int _debug = (state)->debug +# define DO_TRACE (_debug) # define TRACE(v) do { \ if (_debug) { \ printf v; \ @@ -117,6 +119,7 @@ static unsigned int sre_toupper(unsigned int ch) { } while (0) #elif VERBOSE == 2 # define INIT_TRACE(state) +# define DO_TRACE 1 # define TRACE(v) printf v #else # error VERBOSE must be 0, 1 or 2 diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 92dd725c70fd38..f5497d9ff2b93f 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -373,6 +373,19 @@ SRE(count)(SRE_STATE* state, const SRE_CODE* pattern, Py_ssize_t maxcount) state->lastindex = ctx->lastindex; \ } while (0) +#define LAST_PTR_PUSH() \ + do { \ + TRACE(("push last_ptr: %zd", \ + PTR_TO_INDEX(ctx->u.rep->last_ptr))); \ + DATA_PUSH(&ctx->u.rep->last_ptr); \ + } while (0) +#define LAST_PTR_POP() \ + do { \ + DATA_POP(&ctx->u.rep->last_ptr); \ + TRACE(("pop last_ptr: %zd", \ + PTR_TO_INDEX(ctx->u.rep->last_ptr))); \ + } while (0) + #define RETURN_ERROR(i) do { return i; } while(0) #define RETURN_FAILURE do { ret = 0; goto exit; } while(0) #define RETURN_SUCCESS do { ret = 1; goto exit; } while(0) @@ -449,8 +462,27 @@ do { \ #define DATA_LOOKUP_AT(t,p,pos) \ DATA_STACK_LOOKUP_AT(state,t,p,pos) +#define PTR_TO_INDEX(ptr) \ + ((ptr) ? ((char*)(ptr) - (char*)state->beginning) / state->charsize : -1) + +#if VERBOSE +# define MARK_TRACE(label, lastmark) \ + do if (DO_TRACE) { \ + TRACE(("%s %d marks:", (label), (lastmark)+1)); \ + for (int j = 0; j <= (lastmark); j++) { \ + if (j && (j & 1) == 0) { \ + TRACE((" ")); \ + } \ + TRACE((" %zd", PTR_TO_INDEX(state->mark[j]))); \ + } \ + TRACE(("\n")); \ + } while (0) +#else +# define MARK_TRACE(label, lastmark) +#endif #define MARK_PUSH(lastmark) \ do if (lastmark >= 0) { \ + MARK_TRACE("push", (lastmark)); \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_PUSH(state, state->mark, _marks_size); \ } while (0) @@ -458,16 +490,19 @@ do { \ do if (lastmark >= 0) { \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_POP(state, state->mark, _marks_size, 1); \ + MARK_TRACE("pop", (lastmark)); \ } while (0) #define MARK_POP_KEEP(lastmark) \ do if (lastmark >= 0) { \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_POP(state, state->mark, _marks_size, 0); \ + MARK_TRACE("pop keep", (lastmark)); \ } while (0) #define MARK_POP_DISCARD(lastmark) \ do if (lastmark >= 0) { \ size_t _marks_size = (lastmark+1) * sizeof(void*); \ DATA_STACK_POP_DISCARD(state, _marks_size); \ + MARK_TRACE("pop discard", (lastmark)); \ } while (0) #define JUMP_NONE 0 @@ -1150,11 +1185,11 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) LASTMARK_SAVE(); MARK_PUSH(ctx->lastmark); /* zero-width match protection */ - DATA_PUSH(&ctx->u.rep->last_ptr); + LAST_PTR_PUSH(); ctx->u.rep->last_ptr = state->ptr; DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2, ctx->u.rep->pattern+3); - DATA_POP(&ctx->u.rep->last_ptr); + LAST_PTR_POP(); if (ret) { MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); @@ -1235,11 +1270,11 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ctx->u.rep->count = ctx->count; /* zero-width match protection */ - DATA_PUSH(&ctx->u.rep->last_ptr); + LAST_PTR_PUSH(); ctx->u.rep->last_ptr = state->ptr; DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3, ctx->u.rep->pattern+3); - DATA_POP(&ctx->u.rep->last_ptr); + LAST_PTR_POP(); if (ret) { RETURN_ON_ERROR(ret); RETURN_SUCCESS; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index cecc3785c7661c..7bc30cb3529d47 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4133,7 +4133,8 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, if (PyObject_GetBuffer(cadata, &buf, PyBUF_SIMPLE)) { goto error; } - if (!PyBuffer_IsContiguous(&buf, 'C') || buf.ndim > 1) { + assert(PyBuffer_IsContiguous(&buf, 'C')); + if (buf.ndim > 1) { PyBuffer_Release(&buf); PyErr_SetString(PyExc_TypeError, "cadata should be a contiguous buffer with " @@ -5786,52 +5787,44 @@ sslmodule_add_option(PyObject *m, const char *name, uint64_t value) static int sslmodule_init_constants(PyObject *m) { - PyModule_AddStringConstant(m, "_DEFAULT_CIPHERS", - PY_SSL_DEFAULT_CIPHER_STRING); - - PyModule_AddIntConstant(m, "SSL_ERROR_ZERO_RETURN", - PY_SSL_ERROR_ZERO_RETURN); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_READ", - PY_SSL_ERROR_WANT_READ); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_WRITE", - PY_SSL_ERROR_WANT_WRITE); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_X509_LOOKUP", - PY_SSL_ERROR_WANT_X509_LOOKUP); - PyModule_AddIntConstant(m, "SSL_ERROR_SYSCALL", - PY_SSL_ERROR_SYSCALL); - PyModule_AddIntConstant(m, "SSL_ERROR_SSL", - PY_SSL_ERROR_SSL); - PyModule_AddIntConstant(m, "SSL_ERROR_WANT_CONNECT", - PY_SSL_ERROR_WANT_CONNECT); + if (PyModule_AddStringConstant(m, "_DEFAULT_CIPHERS", + PY_SSL_DEFAULT_CIPHER_STRING) < 0) + { + return -1; + } + +#define ADD_INT_CONST(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + + ADD_INT_CONST("SSL_ERROR_ZERO_RETURN", PY_SSL_ERROR_ZERO_RETURN); + ADD_INT_CONST("SSL_ERROR_WANT_READ", PY_SSL_ERROR_WANT_READ); + ADD_INT_CONST("SSL_ERROR_WANT_WRITE", PY_SSL_ERROR_WANT_WRITE); + ADD_INT_CONST("SSL_ERROR_WANT_X509_LOOKUP", PY_SSL_ERROR_WANT_X509_LOOKUP); + ADD_INT_CONST("SSL_ERROR_SYSCALL", PY_SSL_ERROR_SYSCALL); + ADD_INT_CONST("SSL_ERROR_SSL", PY_SSL_ERROR_SSL); + ADD_INT_CONST("SSL_ERROR_WANT_CONNECT", PY_SSL_ERROR_WANT_CONNECT); /* non ssl.h errorcodes */ - PyModule_AddIntConstant(m, "SSL_ERROR_EOF", - PY_SSL_ERROR_EOF); - PyModule_AddIntConstant(m, "SSL_ERROR_INVALID_ERROR_CODE", - PY_SSL_ERROR_INVALID_ERROR_CODE); + ADD_INT_CONST("SSL_ERROR_EOF", PY_SSL_ERROR_EOF); + ADD_INT_CONST("SSL_ERROR_INVALID_ERROR_CODE", + PY_SSL_ERROR_INVALID_ERROR_CODE); /* cert requirements */ - PyModule_AddIntConstant(m, "CERT_NONE", - PY_SSL_CERT_NONE); - PyModule_AddIntConstant(m, "CERT_OPTIONAL", - PY_SSL_CERT_OPTIONAL); - PyModule_AddIntConstant(m, "CERT_REQUIRED", - PY_SSL_CERT_REQUIRED); + ADD_INT_CONST("CERT_NONE", PY_SSL_CERT_NONE); + ADD_INT_CONST("CERT_OPTIONAL", PY_SSL_CERT_OPTIONAL); + ADD_INT_CONST("CERT_REQUIRED", PY_SSL_CERT_REQUIRED); /* CRL verification for verification_flags */ - PyModule_AddIntConstant(m, "VERIFY_DEFAULT", - 0); - PyModule_AddIntConstant(m, "VERIFY_CRL_CHECK_LEAF", - X509_V_FLAG_CRL_CHECK); - PyModule_AddIntConstant(m, "VERIFY_CRL_CHECK_CHAIN", - X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL); - PyModule_AddIntConstant(m, "VERIFY_X509_STRICT", - X509_V_FLAG_X509_STRICT); - PyModule_AddIntConstant(m, "VERIFY_ALLOW_PROXY_CERTS", - X509_V_FLAG_ALLOW_PROXY_CERTS); - PyModule_AddIntConstant(m, "VERIFY_X509_TRUSTED_FIRST", - X509_V_FLAG_TRUSTED_FIRST); + ADD_INT_CONST("VERIFY_DEFAULT", 0); + ADD_INT_CONST("VERIFY_CRL_CHECK_LEAF", X509_V_FLAG_CRL_CHECK); + ADD_INT_CONST("VERIFY_CRL_CHECK_CHAIN", + X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL); + ADD_INT_CONST("VERIFY_X509_STRICT", X509_V_FLAG_X509_STRICT); + ADD_INT_CONST("VERIFY_ALLOW_PROXY_CERTS", X509_V_FLAG_ALLOW_PROXY_CERTS); + ADD_INT_CONST("VERIFY_X509_TRUSTED_FIRST", X509_V_FLAG_TRUSTED_FIRST); #ifdef X509_V_FLAG_PARTIAL_CHAIN - PyModule_AddIntConstant(m, "VERIFY_X509_PARTIAL_CHAIN", - X509_V_FLAG_PARTIAL_CHAIN); + ADD_INT_CONST("VERIFY_X509_PARTIAL_CHAIN", X509_V_FLAG_PARTIAL_CHAIN); #endif /* Alert Descriptions from ssl.h */ @@ -5839,7 +5832,7 @@ sslmodule_init_constants(PyObject *m) /* http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6 */ #define ADD_AD_CONSTANT(s) \ - PyModule_AddIntConstant(m, "ALERT_DESCRIPTION_"#s, \ + ADD_INT_CONST("ALERT_DESCRIPTION_"#s, \ SSL_AD_##s) ADD_AD_CONSTANT(CLOSE_NOTIFY); @@ -5887,23 +5880,15 @@ sslmodule_init_constants(PyObject *m) /* protocol versions */ #ifndef OPENSSL_NO_SSL3 - PyModule_AddIntConstant(m, "PROTOCOL_SSLv3", - PY_SSL_VERSION_SSL3); + ADD_INT_CONST("PROTOCOL_SSLv3", PY_SSL_VERSION_SSL3); #endif - PyModule_AddIntConstant(m, "PROTOCOL_SSLv23", - PY_SSL_VERSION_TLS); - PyModule_AddIntConstant(m, "PROTOCOL_TLS", - PY_SSL_VERSION_TLS); - PyModule_AddIntConstant(m, "PROTOCOL_TLS_CLIENT", - PY_SSL_VERSION_TLS_CLIENT); - PyModule_AddIntConstant(m, "PROTOCOL_TLS_SERVER", - PY_SSL_VERSION_TLS_SERVER); - PyModule_AddIntConstant(m, "PROTOCOL_TLSv1", - PY_SSL_VERSION_TLS1); - PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_1", - PY_SSL_VERSION_TLS1_1); - PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_2", - PY_SSL_VERSION_TLS1_2); + ADD_INT_CONST("PROTOCOL_SSLv23", PY_SSL_VERSION_TLS); + ADD_INT_CONST("PROTOCOL_TLS", PY_SSL_VERSION_TLS); + ADD_INT_CONST("PROTOCOL_TLS_CLIENT", PY_SSL_VERSION_TLS_CLIENT); + ADD_INT_CONST("PROTOCOL_TLS_SERVER", PY_SSL_VERSION_TLS_SERVER); + ADD_INT_CONST("PROTOCOL_TLSv1", PY_SSL_VERSION_TLS1); + ADD_INT_CONST("PROTOCOL_TLSv1_1", PY_SSL_VERSION_TLS1_1); + ADD_INT_CONST("PROTOCOL_TLSv1_2", PY_SSL_VERSION_TLS1_2); #define ADD_OPTION(NAME, VALUE) if (sslmodule_add_option(m, NAME, (VALUE)) < 0) return -1 @@ -5948,50 +5933,52 @@ sslmodule_init_constants(PyObject *m) ADD_OPTION("OP_ENABLE_KTLS", SSL_OP_ENABLE_KTLS); #endif +#undef ADD_OPTION + #ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT - PyModule_AddIntConstant(m, "HOSTFLAG_ALWAYS_CHECK_SUBJECT", - X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); + ADD_INT_CONST("HOSTFLAG_ALWAYS_CHECK_SUBJECT", + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT); #endif #ifdef X509_CHECK_FLAG_NEVER_CHECK_SUBJECT - PyModule_AddIntConstant(m, "HOSTFLAG_NEVER_CHECK_SUBJECT", - X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); + ADD_INT_CONST("HOSTFLAG_NEVER_CHECK_SUBJECT", + X509_CHECK_FLAG_NEVER_CHECK_SUBJECT); #endif #ifdef X509_CHECK_FLAG_NO_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_NO_WILDCARDS", - X509_CHECK_FLAG_NO_WILDCARDS); + ADD_INT_CONST("HOSTFLAG_NO_WILDCARDS", + X509_CHECK_FLAG_NO_WILDCARDS); #endif #ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_NO_PARTIAL_WILDCARDS", - X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + ADD_INT_CONST("HOSTFLAG_NO_PARTIAL_WILDCARDS", + X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); #endif #ifdef X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS - PyModule_AddIntConstant(m, "HOSTFLAG_MULTI_LABEL_WILDCARDS", - X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); + ADD_INT_CONST("HOSTFLAG_MULTI_LABEL_WILDCARDS", + X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS); #endif #ifdef X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS - PyModule_AddIntConstant(m, "HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", - X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); + ADD_INT_CONST("HOSTFLAG_SINGLE_LABEL_SUBDOMAINS", + X509_CHECK_FLAG_SINGLE_LABEL_SUBDOMAINS); #endif /* file types */ - PyModule_AddIntConstant(m, "ENCODING_PEM", PY_SSL_ENCODING_PEM); - PyModule_AddIntConstant(m, "ENCODING_DER", PY_SSL_ENCODING_DER); + ADD_INT_CONST("ENCODING_PEM", PY_SSL_ENCODING_PEM); + ADD_INT_CONST("ENCODING_DER", PY_SSL_ENCODING_DER); /* protocol versions */ - PyModule_AddIntConstant(m, "PROTO_MINIMUM_SUPPORTED", - PY_PROTO_MINIMUM_SUPPORTED); - PyModule_AddIntConstant(m, "PROTO_MAXIMUM_SUPPORTED", - PY_PROTO_MAXIMUM_SUPPORTED); - PyModule_AddIntConstant(m, "PROTO_SSLv3", PY_PROTO_SSLv3); - PyModule_AddIntConstant(m, "PROTO_TLSv1", PY_PROTO_TLSv1); - PyModule_AddIntConstant(m, "PROTO_TLSv1_1", PY_PROTO_TLSv1_1); - PyModule_AddIntConstant(m, "PROTO_TLSv1_2", PY_PROTO_TLSv1_2); - PyModule_AddIntConstant(m, "PROTO_TLSv1_3", PY_PROTO_TLSv1_3); + ADD_INT_CONST("PROTO_MINIMUM_SUPPORTED", PY_PROTO_MINIMUM_SUPPORTED); + ADD_INT_CONST("PROTO_MAXIMUM_SUPPORTED", PY_PROTO_MAXIMUM_SUPPORTED); + ADD_INT_CONST("PROTO_SSLv3", PY_PROTO_SSLv3); + ADD_INT_CONST("PROTO_TLSv1", PY_PROTO_TLSv1); + ADD_INT_CONST("PROTO_TLSv1_1", PY_PROTO_TLSv1_1); + ADD_INT_CONST("PROTO_TLSv1_2", PY_PROTO_TLSv1_2); + ADD_INT_CONST("PROTO_TLSv1_3", PY_PROTO_TLSv1_3); #define addbool(m, key, value) \ do { \ PyObject *bool_obj = (value) ? Py_True : Py_False; \ - PyModule_AddObjectRef((m), (key), bool_obj); \ + if (PyModule_AddObjectRef((m), (key), bool_obj) < 0) { \ + return -1; \ + } \ } while (0) addbool(m, "HAS_SNI", 1); @@ -6032,6 +6019,9 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_TLSv1_3", 0); #endif +#undef addbool +#undef ADD_INT_CONST + return 0; } diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 86d8f5984dd3e5..e4cd15503fd11f 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -138,32 +138,25 @@ getargs_w_star(PyObject *self, PyObject *args) } static PyObject * -test_empty_argparse(PyObject *self, PyObject *Py_UNUSED(ignored)) +getargs_empty(PyObject *self, PyObject *args, PyObject *kwargs) { /* Test that formats can begin with '|'. See issue #4720. */ - PyObject *dict = NULL; - static char *kwlist[] = {NULL}; - PyObject *tuple = PyTuple_New(0); - if (!tuple) { - return NULL; - } + assert(PyTuple_CheckExact(args)); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + int result; - if (!(result = PyArg_ParseTuple(tuple, "|:test_empty_argparse"))) { - goto done; - } - dict = PyDict_New(); - if (!dict) { - goto done; - } - result = PyArg_ParseTupleAndKeywords(tuple, dict, "|:test_empty_argparse", - kwlist); - done: - Py_DECREF(tuple); - Py_XDECREF(dict); + if (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0) { + static char *kwlist[] = {NULL}; + result = PyArg_ParseTupleAndKeywords(args, kwargs, "|:getargs_empty", + kwlist); + } + else { + result = PyArg_ParseTuple(args, "|:getargs_empty"); + } if (!result) { return NULL; } - Py_RETURN_NONE; + return PyLong_FromLong(result); } /* Test tuple argument processing */ @@ -354,75 +347,6 @@ getargs_K(PyObject *self, PyObject *args) return PyLong_FromUnsignedLongLong(value); } -/* This function not only tests the 'k' getargs code, but also the - PyLong_AsUnsignedLongMask() function. */ -static PyObject * -test_k_code(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *tuple, *num; - unsigned long value; - - tuple = PyTuple_New(1); - if (tuple == NULL) { - return NULL; - } - - /* a number larger than ULONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) { - return NULL; - } - - value = PyLong_AsUnsignedLongMask(num); - if (value != ULONG_MAX) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: " - "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = 0; - if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { - return NULL; - } - if (value != ULONG_MAX) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: k code returned wrong value for long 0xFFF...FFF"); - return NULL; - } - - Py_DECREF(num); - num = PyLong_FromString("-FFFFFFFF000000000000000042", NULL, 16); - if (num == NULL) { - return NULL; - } - - value = PyLong_AsUnsignedLongMask(num); - if (value != (unsigned long)-0x42) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: " - "PyLong_AsUnsignedLongMask() returned wrong value for long -0xFFF..000042"); - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = 0; - if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { - return NULL; - } - if (value != (unsigned long)-0x42) { - PyErr_SetString(PyExc_AssertionError, - "test_k_code: k code returned wrong value for long -0xFFF..000042"); - return NULL; - } - - Py_DECREF(tuple); - Py_RETURN_NONE; -} - static PyObject * getargs_f(PyObject *self, PyObject *args) { @@ -703,95 +627,6 @@ getargs_et_hash(PyObject *self, PyObject *args) return result; } -/* Test the L code for PyArg_ParseTuple. This should deliver a long long - for both long and int arguments. The test may leak a little memory if - it fails. -*/ -static PyObject * -test_L_code(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *tuple, *num; - long long value; - - tuple = PyTuple_New(1); - if (tuple == NULL) { - return NULL; - } - - num = PyLong_FromLong(42); - if (num == NULL) { - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = -1; - if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { - return NULL; - } - if (value != 42) { - PyErr_SetString(PyExc_AssertionError, - "test_L_code: L code returned wrong value for long 42"); - return NULL; - } - - Py_DECREF(num); - num = PyLong_FromLong(42); - if (num == NULL) { - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, num); - - value = -1; - if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { - return NULL; - } - if (value != 42) { - PyErr_SetString(PyExc_AssertionError, - "test_L_code: L code returned wrong value for int 42"); - return NULL; - } - - Py_DECREF(tuple); - Py_RETURN_NONE; -} - -/* Test the s and z codes for PyArg_ParseTuple. -*/ -static PyObject * -test_s_code(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - /* Unicode strings should be accepted */ - PyObject *tuple = PyTuple_New(1); - if (tuple == NULL) { - return NULL; - } - - PyObject *obj = PyUnicode_Decode("t\xeate", strlen("t\xeate"), - "latin-1", NULL); - if (obj == NULL) { - return NULL; - } - - PyTuple_SET_ITEM(tuple, 0, obj); - - /* These two blocks used to raise a TypeError: - * "argument must be string without null bytes, not str" - */ - char *value; - if (!PyArg_ParseTuple(tuple, "s:test_s_code1", &value)) { - return NULL; - } - - if (!PyArg_ParseTuple(tuple, "z:test_s_code2", &value)) { - return NULL; - } - - Py_DECREF(tuple); - Py_RETURN_NONE; -} - static PyObject * gh_99240_clear_args(PyObject *self, PyObject *args) { @@ -845,6 +680,7 @@ static PyMethodDef test_methods[] = { {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, {"getargs_w_star", getargs_w_star, METH_VARARGS}, + {"getargs_empty", _PyCFunction_CAST(getargs_empty), METH_VARARGS|METH_KEYWORDS}, {"getargs_y", getargs_y, METH_VARARGS}, {"getargs_y_hash", getargs_y_hash, METH_VARARGS}, {"getargs_y_star", getargs_y_star, METH_VARARGS}, @@ -852,10 +688,6 @@ static PyMethodDef test_methods[] = { {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, - {"test_L_code", test_L_code, METH_NOARGS}, - {"test_empty_argparse", test_empty_argparse, METH_NOARGS}, - {"test_k_code", test_k_code, METH_NOARGS}, - {"test_s_code", test_s_code, METH_NOARGS}, {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index e5dbb9cc49ff4b..4fa77a8844ee4d 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -49,6 +49,7 @@ int _PyTestCapi_Init_PyAtomic(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *mod); +int _PyTestCapi_Init_Sys(PyObject *); int _PyTestCapi_Init_VectorcallLimited(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); diff --git a/Modules/_testcapi/sys.c b/Modules/_testcapi/sys.c new file mode 100644 index 00000000000000..aa40e3cd5b9b29 --- /dev/null +++ b/Modules/_testcapi/sys.c @@ -0,0 +1,56 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + PyObject *result = PySys_GetObject(name); + if (result == NULL) { + result = PyExc_AttributeError; + } + return Py_NewRef(result); +} + +static PyObject * +sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *value; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + return NULL; + } + NULLABLE(value); + RETURN_INT(PySys_SetObject(name, value)); +} + +static PyObject * +sys_getxoptions(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(ignored)) +{ + PyObject *result = PySys_GetXOptions(); + return Py_XNewRef(result); +} + + +static PyMethodDef test_methods[] = { + {"sys_getobject", sys_getobject, METH_O}, + {"sys_setobject", sys_setobject, METH_VARARGS}, + {"sys_getxoptions", sys_getxoptions, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Sys(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index d52d88a65d86fc..a10183dddeca98 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -634,7 +634,7 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) NULLABLE(unicode); s = PyUnicode_AsUTF8AndSize(unicode, &size); if (s == NULL) { - assert(size == UNINITIALIZED_SIZE); + assert(size == -1); return NULL; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 577fea35d97b7e..dc9a25b6c9ff3e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3231,35 +3231,6 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } -static PyObject * -sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) -{ - const char *name; - Py_ssize_t size; - if (!PyArg_Parse(arg, "z#", &name, &size)) { - return NULL; - } - PyObject *result = PySys_GetObject(name); - if (result == NULL) { - result = PyExc_AttributeError; - } - return Py_NewRef(result); -} - -static PyObject * -sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) -{ - const char *name; - Py_ssize_t size; - PyObject *value; - if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { - return NULL; - } - NULLABLE(value); - RETURN_INT(PySys_SetObject(name, value)); -} - - static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3392,8 +3363,6 @@ static PyMethodDef TestMethods[] = { {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, - {"sys_getobject", sys_getobject, METH_O}, - {"sys_setobject", sys_setobject, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -4038,6 +4007,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Sys(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 31f9a5a57d22cc..1869f48c2b1fbf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1002,6 +1002,32 @@ get_executor(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, ioffset); } +static PyObject * +add_executor_dependency(PyObject *self, PyObject *args) +{ + PyObject *exec; + PyObject *obj; + if (!PyArg_ParseTuple(args, "OO", &exec, &obj)) { + return NULL; + } + /* No way to tell in general if exec is an executor, so we only accept + * counting_executor */ + if (strcmp(Py_TYPE(exec)->tp_name, "counting_executor")) { + PyErr_SetString(PyExc_TypeError, "argument must be a counting_executor"); + return NULL; + } + _Py_Executor_DependsOn((_PyExecutorObject *)exec, obj); + Py_RETURN_NONE; +} + +static PyObject * +invalidate_executors(PyObject *self, PyObject *obj) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _Py_Executors_InvalidateDependency(interp, obj); + Py_RETURN_NONE; +} + static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ @@ -1595,6 +1621,8 @@ static PyMethodDef module_functions[] = { {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, + {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, + {"invalidate_executors", invalidate_executors, METH_O, NULL}, {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 33b49dacaa946e..82a0c827deeddf 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -75,10 +75,18 @@ test_lock_two_threads(PyObject *self, PyObject *obj) assert(test_data.m.v == 1); PyThread_start_new_thread(lock_thread, &test_data); - while (!_Py_atomic_load_int(&test_data.started)) { - pysleep(10); - } - pysleep(10); // allow some time for the other thread to try to lock + + // wait up to two seconds for the lock_thread to attempt to lock "m" + int iters = 0; + uint8_t v; + do { + pysleep(10); // allow some time for the other thread to try to lock + v = _Py_atomic_load_uint8_relaxed(&test_data.m.v); + assert(v == 1 || v == 3); + iters++; + } while (v != 3 && iters < 200); + + // both the "locked" and the "has parked" bits should be set assert(test_data.m.v == 3); PyMutex_Unlock(&test_data.m); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 4d453040503643..a849a200df625c 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -3,7 +3,6 @@ /* Interface to Sjoerd's portable C thread library */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_interp.h" // _PyInterpreterState.threads.count #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_WriteUnraisableMsg() diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eec33499b983fe..8c48b6f3ec6ef6 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -212,7 +212,7 @@ class DWORD_return_converter(CReturnConverter): self.declare(data) self.err_occurred_if("_return_value == PY_DWORD_MAX", data) data.return_conversion.append( - 'return_value = Py_BuildValue("k", _return_value);\n') + 'return_value = PyLong_FromUnsignedLong(_return_value);\n') class LPVOID_return_converter(CReturnConverter): type = 'LPVOID' @@ -223,7 +223,7 @@ class LPVOID_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = HANDLE_TO_PYNUM(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=011ee0c3a2244bfe]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ef52a757a1830d92]*/ #include "clinic/_winapi.c.h" diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 3f14dd13988da9..97e550197a2a8d 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -816,7 +816,7 @@ zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, /*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { if (from_cache) { - PyObject *val_args = Py_BuildValue("(O)", key); + PyObject *val_args = PyTuple_Pack(1, key); if (val_args == NULL) { return NULL; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index cf74690f41288c..b97ade6126fa08 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,7 +11,6 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_long.h" // _PyLong_FromByteArray() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() diff --git a/Modules/binascii.c b/Modules/binascii.c index a87a2ef2e89927..17970aa5e9456c 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -185,13 +185,7 @@ ascii_buffer_converter(PyObject *arg, Py_buffer *buf) "not '%.100s'", Py_TYPE(arg)->tp_name); return 0; } - if (!PyBuffer_IsContiguous(buf, 'C')) { - PyErr_Format(PyExc_TypeError, - "argument should be a contiguous buffer, " - "not '%.100s'", Py_TYPE(arg)->tp_name); - PyBuffer_Release(buf); - return 0; - } + assert(PyBuffer_IsContiguous(buf, 'C')); return Py_CLEANUP_SUPPORTED; } diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 4f2b91f3b19b16..305ade17b1f1aa 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -154,10 +154,6 @@ _multibytecodec_MultibyteCodec_decode(MultibyteCodecObject *self, PyObject *cons if (PyObject_GetBuffer(args[0], &input, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&input, 'C')) { - _PyArg_BadArgument("decode", "argument 'input'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -374,10 +370,6 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb if (PyObject_GetBuffer(args[0], &input, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&input, 'C')) { - _PyArg_BadArgument("decode", "argument 'input'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -690,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=38f8d42721eea1e6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=219a363662d2fbff input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 7c1da9004549a6..5d3c16a98423ba 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -9,7 +9,6 @@ #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_FromByteArray() #include "multibytecodec.h" #include "clinic/multibytecodec.c.h" diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index b665c01f1a9ff9..de7b3993596446 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -35,10 +35,6 @@ _bz2_BZ2Compressor_compress(BZ2Compressor *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _bz2_BZ2Compressor_compress_impl(self, &data); exit: @@ -181,10 +177,6 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -242,4 +234,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=90f7b5c451c0a8bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8daa62f47cc4853d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index d69b6824fb7074..12fea806ab5209 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -303,10 +303,6 @@ _codecs_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("escape_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -422,10 +418,6 @@ _codecs_utf_7_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_7_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -492,10 +484,6 @@ _codecs_utf_8_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_8_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -562,10 +550,6 @@ _codecs_utf_16_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -632,10 +616,6 @@ _codecs_utf_16_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_le_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -702,10 +682,6 @@ _codecs_utf_16_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_be_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -774,10 +750,6 @@ _codecs_utf_16_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_16_ex_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -851,10 +823,6 @@ _codecs_utf_32_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -921,10 +889,6 @@ _codecs_utf_32_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_le_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -991,10 +955,6 @@ _codecs_utf_32_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_be_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1063,10 +1023,6 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("utf_32_ex_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1149,10 +1105,6 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("unicode_escape_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -1229,10 +1181,6 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("raw_unicode_escape_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -1299,10 +1247,6 @@ _codecs_latin_1_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("latin_1_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1361,10 +1305,6 @@ _codecs_ascii_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("ascii_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1424,10 +1364,6 @@ _codecs_charmap_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("charmap_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1493,10 +1429,6 @@ _codecs_mbcs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("mbcs_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1567,10 +1499,6 @@ _codecs_oem_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("oem_decode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1646,10 +1574,6 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[1], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("code_page_decode", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -1726,10 +1650,6 @@ _codecs_readbuffer_encode(PyObject *module, PyObject *const *args, Py_ssize_t na if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("readbuffer_encode", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (nargs < 2) { goto skip_optional; @@ -2818,4 +2738,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=40cf63bf2da18359 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d8d9e372f7ccba35 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index dd9f1ab6841d01..f7e0aaf7b23649 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -3163,7 +3163,7 @@ PyDoc_STRVAR(_curses_napms__doc__, #define _CURSES_NAPMS_METHODDEF \ {"napms", (PyCFunction)_curses_napms, METH_O, _curses_napms__doc__}, -static PyObject * +static int _curses_napms_impl(PyObject *module, int ms); static PyObject * @@ -3171,12 +3171,17 @@ _curses_napms(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int ms; + int _return_value; ms = PyLong_AsInt(arg); if (ms == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_napms_impl(module, ms); + _return_value = _curses_napms_impl(module, ms); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -4313,4 +4318,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=24ad16254d1eef9c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96887782374f070a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index dc28aaca1f7e07..58650dff288444 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1290,17 +1290,9 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (PyObject_GetBuffer(args[1], &password, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&password, 'C')) { - _PyArg_BadArgument("pbkdf2_hmac", "argument 'password'", "contiguous buffer", args[1]); - goto exit; - } if (PyObject_GetBuffer(args[2], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("pbkdf2_hmac", "argument 'salt'", "contiguous buffer", args[2]); - goto exit; - } iterations = PyLong_AsLong(args[3]); if (iterations == -1 && PyErr_Occurred()) { goto exit; @@ -1388,10 +1380,6 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &password, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&password, 'C')) { - _PyArg_BadArgument("scrypt", "argument 'password'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -1399,10 +1387,6 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[1], &salt, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&salt, 'C')) { - _PyArg_BadArgument("scrypt", "argument 'salt'", "contiguous buffer", args[1]); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -1522,17 +1506,9 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("hmac_digest", "argument 'key'", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &msg, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&msg, 'C')) { - _PyArg_BadArgument("hmac_digest", "argument 'msg'", "contiguous buffer", args[1]); - goto exit; - } digest = args[2]; return_value = _hashlib_hmac_singleshot_impl(module, &key, &msg, digest); @@ -1604,10 +1580,6 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO if (PyObject_GetBuffer(args[0], &key, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&key, 'C')) { - _PyArg_BadArgument("hmac_new", "argument 'key'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -1852,4 +1824,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=21ad88d46922dc00 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b7eddeb3d6ccdeec input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 1079b409a6b6d0..51fab5eab3f7dc 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -35,10 +35,6 @@ _lzma_LZMACompressor_compress(Compressor *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _lzma_LZMACompressor_compress_impl(self, &data); exit: @@ -139,10 +135,6 @@ _lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -325,10 +317,6 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz if (PyObject_GetBuffer(args[1], &encoded_props, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&encoded_props, 'C')) { - _PyArg_BadArgument("_decode_filter_properties", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _lzma__decode_filter_properties_impl(module, filter_id, &encoded_props); exit: @@ -339,4 +327,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=fca7d2b5800dc4c1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e79c05ace76dc96 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index bc692518e5448c..88401b0490a1bb 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -237,10 +237,6 @@ _ssl__SSLSocket_write(PySSLSocket *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _ssl__SSLSocket_write_impl(self, &b); exit: @@ -530,10 +526,6 @@ _ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) if (PyObject_GetBuffer(arg, &protos, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&protos, 'C')) { - _PyArg_BadArgument("_set_alpn_protocols", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _ssl__SSLContext__set_alpn_protocols_impl(self, &protos); exit: @@ -1108,10 +1100,6 @@ _ssl_MemoryBIO_write(PySSLMemoryBIO *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&b, 'C')) { - _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); - goto exit; - } return_value = _ssl_MemoryBIO_write_impl(self, &b); exit: @@ -1180,10 +1168,6 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&view, 'C')) { - _PyArg_BadArgument("RAND_add", "argument 1", "contiguous buffer", args[0]); - goto exit; - } } if (PyFloat_CheckExact(args[1])) { entropy = PyFloat_AS_DOUBLE(args[1]); @@ -1543,4 +1527,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=f15635b2faa3b2db input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aa6b0a898b6077fe input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 45d90710c54865..e5118fbdb3b9d3 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -94,10 +94,6 @@ Struct_unpack(PyStructObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack", "argument", "contiguous buffer", arg); - goto exit; - } return_value = Struct_unpack_impl(self, &buffer); exit: @@ -170,10 +166,6 @@ Struct_unpack_from(PyStructObject *self, PyObject *const *args, Py_ssize_t nargs if (PyObject_GetBuffer(args[0], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack_from", "argument 'buffer'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -300,10 +292,6 @@ unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = unpack_impl(module, s_object, &buffer); exit: @@ -379,10 +367,6 @@ unpack_from(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (PyObject_GetBuffer(args[1], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("unpack_from", "argument 'buffer'", "contiguous buffer", args[1]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -452,4 +436,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=1749aaf639d5c11c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6a20e87f9b298b14 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index ae8cc1d6283217..3a3231c051ef71 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -568,7 +568,7 @@ _winapi_GetExitCodeProcess(PyObject *module, PyObject *arg) if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -595,7 +595,7 @@ _winapi_GetLastError(PyObject *module, PyObject *Py_UNUSED(ignored)) if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -1305,7 +1305,7 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -1479,4 +1479,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=aaf29735c47f55fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e1a9908bb82a6379 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index bff9fd3c87b646..dbce0313541649 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -442,10 +442,6 @@ array_array_frombytes(arrayobject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("frombytes", "argument", "contiguous buffer", arg); - goto exit; - } return_value = array_array_frombytes_impl(self, &buffer); exit: @@ -671,4 +667,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=d58693e1157540ef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf086c01e7e482bf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index b80983ca9a1709..1adca415dfee12 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -92,10 +92,6 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_uu", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -243,10 +239,6 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_base64", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -291,10 +283,6 @@ binascii_crc_hqx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("crc_hqx", "argument 1", "contiguous buffer", args[0]); - goto exit; - } crc = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); if (crc == (unsigned int)-1 && PyErr_Occurred()) { goto exit; @@ -336,10 +324,6 @@ binascii_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("crc32", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -435,10 +419,6 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_hex", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -528,10 +508,6 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("hexlify", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -755,10 +731,6 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("b2a_qp", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -795,4 +767,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=3259f3b018abee96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=968767b663ed889d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 8e1d719e4433be..8b285e4a8f0a72 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -624,10 +624,6 @@ _overlapped_Overlapped_ReadFileInto(OverlappedObject *self, PyObject *const *arg if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("ReadFileInto", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _overlapped_Overlapped_ReadFileInto_impl(self, handle, &bufobj); exit: @@ -715,10 +711,6 @@ _overlapped_Overlapped_WSARecvInto(OverlappedObject *self, PyObject *const *args if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSARecvInto", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } @@ -763,10 +755,6 @@ _overlapped_Overlapped_WriteFile(OverlappedObject *self, PyObject *const *args, if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WriteFile", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = _overlapped_Overlapped_WriteFile_impl(self, handle, &bufobj); exit: @@ -809,10 +797,6 @@ _overlapped_Overlapped_WSASend(OverlappedObject *self, PyObject *const *args, Py if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSASend", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } @@ -1138,10 +1122,6 @@ _overlapped_Overlapped_WSASendTo(OverlappedObject *self, PyObject *const *args, if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSASendTo", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } @@ -1239,10 +1219,6 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bufobj, 'C')) { - _PyArg_BadArgument("WSARecvFromInto", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!_PyLong_UnsignedLong_Converter(args[2], &size)) { goto exit; } @@ -1263,4 +1239,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=85884c2341fcbef7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=958cbddbcc355f47 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 29c90c5c261b57..9473dd70ff1460 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7208,10 +7208,6 @@ os_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("write", "argument 2", "contiguous buffer", args[1]); - goto exit; - } _return_value = os_write_impl(module, fd, &data); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -7810,10 +7806,6 @@ os_pwrite(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[1], &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&buffer, 'C')) { - _PyArg_BadArgument("pwrite", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (!Py_off_t_converter(args[2], &offset)) { goto exit; } @@ -10252,10 +10244,6 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (PyObject_GetBuffer(args[2], &value, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&value, 'C')) { - _PyArg_BadArgument("setxattr", "argument 'value'", "contiguous buffer", args[2]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -12415,4 +12403,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=274174066fff3256 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a05abdc48e3def44 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 850a4cd965267f..6b09abe309bf48 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -70,10 +70,6 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -164,10 +160,6 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -334,10 +326,6 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[5], &zdict, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&zdict, 'C')) { - _PyArg_BadArgument("compressobj", "argument 'zdict'", "contiguous buffer", args[5]); - goto exit; - } skip_optional_pos: return_value = zlib_compressobj_impl(module, level, method, wbits, memLevel, strategy, &zdict); @@ -473,10 +461,6 @@ zlib_Compress_compress(compobject *self, PyTypeObject *cls, PyObject *const *arg if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("compress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } return_value = zlib_Compress_compress_impl(self, cls, &data); exit: @@ -553,10 +537,6 @@ zlib_Decompress_decompress(compobject *self, PyTypeObject *cls, PyObject *const if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -965,10 +945,6 @@ zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -1026,10 +1002,6 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("adler32", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1080,10 +1052,6 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data, 'C')) { - _PyArg_BadArgument("crc32", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1130,4 +1098,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=bd96ba786b0d8d42 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6dd97dc851c39031 input=a9049054013a1b77]*/ diff --git a/Modules/main.c b/Modules/main.c index b5ee34d0141daf..df2ce550245088 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -249,7 +249,7 @@ pymain_run_command(wchar_t *command) PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags |= PyCF_IGNORE_COOKIE; - ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), &cf); + ret = _PyRun_SimpleStringFlagsWithName(PyBytes_AsString(bytes), "", &cf); Py_DECREF(bytes); return (ret != 0); diff --git a/Modules/overlapped.c b/Modules/overlapped.c index e23db22dadb18b..fd40e91d0f50c4 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -599,8 +599,7 @@ _overlapped_FormatMessage_impl(PyObject *module, DWORD code) if (n) { while (iswspace(lpMsgBuf[n-1])) --n; - lpMsgBuf[n] = L'\0'; - res = Py_BuildValue("u", lpMsgBuf); + res = PyUnicode_FromWideChar(lpMsgBuf, n); } else { res = PyUnicode_FromFormat("unknown error code %u", code); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 650ae4bbd68656..8611f8f73aff05 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11276,9 +11276,9 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, done: #if !defined(HAVE_LARGEFILE_SUPPORT) - return Py_BuildValue("l", sbytes); + return PyLong_FromLong(sbytes); #else - return Py_BuildValue("L", sbytes); + return PyLong_FromLongLong(sbytes); #endif #else @@ -11291,7 +11291,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (ret < 0) return (!async_err) ? posix_error() : NULL; - return Py_BuildValue("n", ret); + return PyLong_FromSsize_t(ret); } #endif off_t offset; @@ -11312,7 +11312,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, return (!async_err) ? posix_error() : NULL; if (offset >= st.st_size) { - return Py_BuildValue("i", 0); + return PyLong_FromLong(0); } // On illumos specifically sendfile() may perform a partial write but @@ -11338,7 +11338,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (ret < 0) return (!async_err) ? posix_error() : NULL; - return Py_BuildValue("n", ret); + return PyLong_FromSsize_t(ret); #endif } #endif /* HAVE_SENDFILE */ @@ -14805,6 +14805,7 @@ typedef struct { #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; uint64_t win32_file_index; + uint64_t win32_file_index_high; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -15134,11 +15135,10 @@ os_DirEntry_inode_impl(DirEntry *self) } self->win32_file_index = stat.st_ino; + self->win32_file_index_high = stat.st_ino_high; self->got_file_index = 1; } - static_assert(sizeof(unsigned long long) >= sizeof(self->win32_file_index), - "DirEntry.win32_file_index is larger than unsigned long long"); - return PyLong_FromUnsignedLongLong(self->win32_file_index); + return _pystat_l128_from_l64_l64(self->win32_file_index, self->win32_file_index_high); #else /* POSIX */ static_assert(sizeof(unsigned long long) >= sizeof(self->d_ino), "DirEntry.d_ino is larger than unsigned long long"); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index bd24523eac830b..21579a80dd7f70 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -895,7 +895,7 @@ static PyObject * pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) /*[clinic end generated code: output=2886cb21f9a8739a input=918d71c38009620e]*/ { - return Py_BuildValue("z", XML_GetBase(self->itself)); + return conv_string_to_unicode(XML_GetBase(self->itself)); } /*[clinic input] @@ -1585,7 +1585,7 @@ static PyObject * pyexpat_ErrorString_impl(PyObject *module, long code) /*[clinic end generated code: output=2feae50d166f2174 input=cc67de010d9e62b3]*/ { - return Py_BuildValue("z", XML_ErrorString((int)code)); + return conv_string_to_unicode(XML_ErrorString((int)code)); } /* List of methods defined in the module */ diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index c56e682b21e2a1..80330d2f88f8bc 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2444,12 +2444,18 @@ _select_exec(PyObject *m) return -1; } +#define ADD_INT(VAL) do { \ + if (PyModule_AddIntConstant((m), #VAL, (VAL)) < 0) { \ + return -1; \ + } \ +} while (0) + #ifdef PIPE_BUF #ifdef HAVE_BROKEN_PIPE_BUF #undef PIPE_BUF #define PIPE_BUF 512 #endif - PyModule_AddIntMacro(m, PIPE_BUF); + ADD_INT(PIPE_BUF); #endif #if defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL) @@ -2468,31 +2474,31 @@ _select_exec(PyObject *m) return -1; } - PyModule_AddIntMacro(m, POLLIN); - PyModule_AddIntMacro(m, POLLPRI); - PyModule_AddIntMacro(m, POLLOUT); - PyModule_AddIntMacro(m, POLLERR); - PyModule_AddIntMacro(m, POLLHUP); - PyModule_AddIntMacro(m, POLLNVAL); + ADD_INT(POLLIN); + ADD_INT(POLLPRI); + ADD_INT(POLLOUT); + ADD_INT(POLLERR); + ADD_INT(POLLHUP); + ADD_INT(POLLNVAL); #ifdef POLLRDNORM - PyModule_AddIntMacro(m, POLLRDNORM); + ADD_INT(POLLRDNORM); #endif #ifdef POLLRDBAND - PyModule_AddIntMacro(m, POLLRDBAND); + ADD_INT(POLLRDBAND); #endif #ifdef POLLWRNORM - PyModule_AddIntMacro(m, POLLWRNORM); + ADD_INT(POLLWRNORM); #endif #ifdef POLLWRBAND - PyModule_AddIntMacro(m, POLLWRBAND); + ADD_INT(POLLWRBAND); #endif #ifdef POLLMSG - PyModule_AddIntMacro(m, POLLMSG); + ADD_INT(POLLMSG); #endif #ifdef POLLRDHUP /* Kernel 2.6.17+ */ - PyModule_AddIntMacro(m, POLLRDHUP); + ADD_INT(POLLRDHUP); #endif } #endif /* HAVE_POLL */ @@ -2515,45 +2521,54 @@ _select_exec(PyObject *m) return -1; } - PyModule_AddIntMacro(m, EPOLLIN); - PyModule_AddIntMacro(m, EPOLLOUT); - PyModule_AddIntMacro(m, EPOLLPRI); - PyModule_AddIntMacro(m, EPOLLERR); - PyModule_AddIntMacro(m, EPOLLHUP); + ADD_INT(EPOLLIN); + ADD_INT(EPOLLOUT); + ADD_INT(EPOLLPRI); + ADD_INT(EPOLLERR); + ADD_INT(EPOLLHUP); #ifdef EPOLLRDHUP /* Kernel 2.6.17 */ - PyModule_AddIntMacro(m, EPOLLRDHUP); + ADD_INT(EPOLLRDHUP); #endif - PyModule_AddIntMacro(m, EPOLLET); + ADD_INT(EPOLLET); #ifdef EPOLLONESHOT /* Kernel 2.6.2+ */ - PyModule_AddIntMacro(m, EPOLLONESHOT); + ADD_INT(EPOLLONESHOT); #endif #ifdef EPOLLEXCLUSIVE - PyModule_AddIntMacro(m, EPOLLEXCLUSIVE); + ADD_INT(EPOLLEXCLUSIVE); #endif #ifdef EPOLLRDNORM - PyModule_AddIntMacro(m, EPOLLRDNORM); + ADD_INT(EPOLLRDNORM); #endif #ifdef EPOLLRDBAND - PyModule_AddIntMacro(m, EPOLLRDBAND); + ADD_INT(EPOLLRDBAND); #endif #ifdef EPOLLWRNORM - PyModule_AddIntMacro(m, EPOLLWRNORM); + ADD_INT(EPOLLWRNORM); #endif #ifdef EPOLLWRBAND - PyModule_AddIntMacro(m, EPOLLWRBAND); + ADD_INT(EPOLLWRBAND); #endif #ifdef EPOLLMSG - PyModule_AddIntMacro(m, EPOLLMSG); + ADD_INT(EPOLLMSG); #endif #ifdef EPOLL_CLOEXEC - PyModule_AddIntMacro(m, EPOLL_CLOEXEC); + ADD_INT(EPOLL_CLOEXEC); #endif #endif /* HAVE_EPOLL */ +#undef ADD_INT + +#define ADD_INT_CONST(NAME, VAL) \ + do { \ + if (PyModule_AddIntConstant(m, NAME, VAL) < 0) { \ + return -1; \ + } \ + } while (0) + #ifdef HAVE_KQUEUE state->kqueue_event_Type = (PyTypeObject *)PyType_FromModuleAndSpec( m, &kqueue_event_Type_spec, NULL); @@ -2574,80 +2589,83 @@ _select_exec(PyObject *m) } /* event filters */ - PyModule_AddIntConstant(m, "KQ_FILTER_READ", EVFILT_READ); - PyModule_AddIntConstant(m, "KQ_FILTER_WRITE", EVFILT_WRITE); + ADD_INT_CONST("KQ_FILTER_READ", EVFILT_READ); + ADD_INT_CONST("KQ_FILTER_WRITE", EVFILT_WRITE); #ifdef EVFILT_AIO - PyModule_AddIntConstant(m, "KQ_FILTER_AIO", EVFILT_AIO); + ADD_INT_CONST("KQ_FILTER_AIO", EVFILT_AIO); #endif #ifdef EVFILT_VNODE - PyModule_AddIntConstant(m, "KQ_FILTER_VNODE", EVFILT_VNODE); + ADD_INT_CONST("KQ_FILTER_VNODE", EVFILT_VNODE); #endif #ifdef EVFILT_PROC - PyModule_AddIntConstant(m, "KQ_FILTER_PROC", EVFILT_PROC); + ADD_INT_CONST("KQ_FILTER_PROC", EVFILT_PROC); #endif #ifdef EVFILT_NETDEV - PyModule_AddIntConstant(m, "KQ_FILTER_NETDEV", EVFILT_NETDEV); + ADD_INT_CONST("KQ_FILTER_NETDEV", EVFILT_NETDEV); #endif #ifdef EVFILT_SIGNAL - PyModule_AddIntConstant(m, "KQ_FILTER_SIGNAL", EVFILT_SIGNAL); + ADD_INT_CONST("KQ_FILTER_SIGNAL", EVFILT_SIGNAL); #endif - PyModule_AddIntConstant(m, "KQ_FILTER_TIMER", EVFILT_TIMER); + ADD_INT_CONST("KQ_FILTER_TIMER", EVFILT_TIMER); /* event flags */ - PyModule_AddIntConstant(m, "KQ_EV_ADD", EV_ADD); - PyModule_AddIntConstant(m, "KQ_EV_DELETE", EV_DELETE); - PyModule_AddIntConstant(m, "KQ_EV_ENABLE", EV_ENABLE); - PyModule_AddIntConstant(m, "KQ_EV_DISABLE", EV_DISABLE); - PyModule_AddIntConstant(m, "KQ_EV_ONESHOT", EV_ONESHOT); - PyModule_AddIntConstant(m, "KQ_EV_CLEAR", EV_CLEAR); + ADD_INT_CONST("KQ_EV_ADD", EV_ADD); + ADD_INT_CONST("KQ_EV_DELETE", EV_DELETE); + ADD_INT_CONST("KQ_EV_ENABLE", EV_ENABLE); + ADD_INT_CONST("KQ_EV_DISABLE", EV_DISABLE); + ADD_INT_CONST("KQ_EV_ONESHOT", EV_ONESHOT); + ADD_INT_CONST("KQ_EV_CLEAR", EV_CLEAR); #ifdef EV_SYSFLAGS - PyModule_AddIntConstant(m, "KQ_EV_SYSFLAGS", EV_SYSFLAGS); + ADD_INT_CONST("KQ_EV_SYSFLAGS", EV_SYSFLAGS); #endif #ifdef EV_FLAG1 - PyModule_AddIntConstant(m, "KQ_EV_FLAG1", EV_FLAG1); + ADD_INT_CONST("KQ_EV_FLAG1", EV_FLAG1); #endif - PyModule_AddIntConstant(m, "KQ_EV_EOF", EV_EOF); - PyModule_AddIntConstant(m, "KQ_EV_ERROR", EV_ERROR); + ADD_INT_CONST("KQ_EV_EOF", EV_EOF); + ADD_INT_CONST("KQ_EV_ERROR", EV_ERROR); /* READ WRITE filter flag */ #ifdef NOTE_LOWAT - PyModule_AddIntConstant(m, "KQ_NOTE_LOWAT", NOTE_LOWAT); + ADD_INT_CONST("KQ_NOTE_LOWAT", NOTE_LOWAT); #endif /* VNODE filter flags */ #ifdef EVFILT_VNODE - PyModule_AddIntConstant(m, "KQ_NOTE_DELETE", NOTE_DELETE); - PyModule_AddIntConstant(m, "KQ_NOTE_WRITE", NOTE_WRITE); - PyModule_AddIntConstant(m, "KQ_NOTE_EXTEND", NOTE_EXTEND); - PyModule_AddIntConstant(m, "KQ_NOTE_ATTRIB", NOTE_ATTRIB); - PyModule_AddIntConstant(m, "KQ_NOTE_LINK", NOTE_LINK); - PyModule_AddIntConstant(m, "KQ_NOTE_RENAME", NOTE_RENAME); - PyModule_AddIntConstant(m, "KQ_NOTE_REVOKE", NOTE_REVOKE); + ADD_INT_CONST("KQ_NOTE_DELETE", NOTE_DELETE); + ADD_INT_CONST("KQ_NOTE_WRITE", NOTE_WRITE); + ADD_INT_CONST("KQ_NOTE_EXTEND", NOTE_EXTEND); + ADD_INT_CONST("KQ_NOTE_ATTRIB", NOTE_ATTRIB); + ADD_INT_CONST("KQ_NOTE_LINK", NOTE_LINK); + ADD_INT_CONST("KQ_NOTE_RENAME", NOTE_RENAME); + ADD_INT_CONST("KQ_NOTE_REVOKE", NOTE_REVOKE); #endif /* PROC filter flags */ #ifdef EVFILT_PROC - PyModule_AddIntConstant(m, "KQ_NOTE_EXIT", NOTE_EXIT); - PyModule_AddIntConstant(m, "KQ_NOTE_FORK", NOTE_FORK); - PyModule_AddIntConstant(m, "KQ_NOTE_EXEC", NOTE_EXEC); - PyModule_AddIntConstant(m, "KQ_NOTE_PCTRLMASK", NOTE_PCTRLMASK); - PyModule_AddIntConstant(m, "KQ_NOTE_PDATAMASK", NOTE_PDATAMASK); - - PyModule_AddIntConstant(m, "KQ_NOTE_TRACK", NOTE_TRACK); - PyModule_AddIntConstant(m, "KQ_NOTE_CHILD", NOTE_CHILD); - PyModule_AddIntConstant(m, "KQ_NOTE_TRACKERR", NOTE_TRACKERR); + ADD_INT_CONST("KQ_NOTE_EXIT", NOTE_EXIT); + ADD_INT_CONST("KQ_NOTE_FORK", NOTE_FORK); + ADD_INT_CONST("KQ_NOTE_EXEC", NOTE_EXEC); + ADD_INT_CONST("KQ_NOTE_PCTRLMASK", NOTE_PCTRLMASK); + ADD_INT_CONST("KQ_NOTE_PDATAMASK", NOTE_PDATAMASK); + + ADD_INT_CONST("KQ_NOTE_TRACK", NOTE_TRACK); + ADD_INT_CONST("KQ_NOTE_CHILD", NOTE_CHILD); + ADD_INT_CONST("KQ_NOTE_TRACKERR", NOTE_TRACKERR); #endif /* NETDEV filter flags */ #ifdef EVFILT_NETDEV - PyModule_AddIntConstant(m, "KQ_NOTE_LINKUP", NOTE_LINKUP); - PyModule_AddIntConstant(m, "KQ_NOTE_LINKDOWN", NOTE_LINKDOWN); - PyModule_AddIntConstant(m, "KQ_NOTE_LINKINV", NOTE_LINKINV); + ADD_INT_CONST("KQ_NOTE_LINKUP", NOTE_LINKUP); + ADD_INT_CONST("KQ_NOTE_LINKDOWN", NOTE_LINKDOWN); + ADD_INT_CONST("KQ_NOTE_LINKINV", NOTE_LINKINV); #endif #endif /* HAVE_KQUEUE */ + +#undef ADD_INT_CONST + return 0; } diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index bc5cdf0ab6b5f2..2932d94858afde 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -657,7 +657,7 @@ signal_strsignal_impl(PyObject *module, int signalnum) Py_RETURN_NONE; #endif - return Py_BuildValue("s", res); + return PyUnicode_FromString(res); } #ifdef HAVE_SIGINTERRUPT diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 3d099d41d1e761..1cf6fa4a8936e9 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -107,7 +107,6 @@ Local naming conventions: #include "Python.h" #include "pycore_capsule.h" // _PyCapsule_SetTraverse() -#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState @@ -7723,10 +7722,10 @@ socket_exec(PyObject *m) /* FreeBSD divert(4) */ #ifdef PF_DIVERT - PyModule_AddIntMacro(m, PF_DIVERT); + ADD_INT_MACRO(m, PF_DIVERT); #endif #ifdef AF_DIVERT - PyModule_AddIntMacro(m, AF_DIVERT); + ADD_INT_MACRO(m, AF_DIVERT); #endif #ifdef AF_PACKET diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 6a872a285d289b..bf48c89f343948 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1740,6 +1740,12 @@ get_gmtoff(time_t t, struct tm *p) static int init_timezone(PyObject *m) { +#define ADD_INT(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + assert(!PyErr_Occurred()); /* This code moved from PyInit_time wholesale to allow calling it from @@ -1763,13 +1769,13 @@ init_timezone(PyObject *m) #if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); #endif - PyModule_AddIntConstant(m, "timezone", _Py_timezone); + ADD_INT("timezone", _Py_timezone); #ifdef HAVE_ALTZONE - PyModule_AddIntConstant(m, "altzone", altzone); + ADD_INT("altzone", altzone); #else - PyModule_AddIntConstant(m, "altzone", _Py_timezone-3600); + ADD_INT("altzone", _Py_timezone-3600); #endif - PyModule_AddIntConstant(m, "daylight", _Py_daylight); + ADD_INT("daylight", _Py_daylight); #ifdef MS_WINDOWS TIME_ZONE_INFORMATION tzinfo = {0}; GetTimeZoneInformation(&tzinfo); @@ -1828,20 +1834,21 @@ init_timezone(PyObject *m) PyObject *tzname_obj; if (janzone < julyzone) { /* DST is reversed in the southern hemisphere */ - PyModule_AddIntConstant(m, "timezone", julyzone); - PyModule_AddIntConstant(m, "altzone", janzone); - PyModule_AddIntConstant(m, "daylight", janzone != julyzone); + ADD_INT("timezone", julyzone); + ADD_INT("altzone", janzone); + ADD_INT("daylight", janzone != julyzone); tzname_obj = Py_BuildValue("(zz)", julyname, janname); } else { - PyModule_AddIntConstant(m, "timezone", janzone); - PyModule_AddIntConstant(m, "altzone", julyzone); - PyModule_AddIntConstant(m, "daylight", janzone != julyzone); + ADD_INT("timezone", janzone); + ADD_INT("altzone", julyzone); + ADD_INT("daylight", janzone != julyzone); tzname_obj = Py_BuildValue("(zz)", janname, julyname); } if (PyModule_Add(m, "tzname", tzname_obj) < 0) { return -1; } #endif // !HAVE_DECL_TZNAME +#undef ADD_INT if (PyErr_Occurred()) { return -1; diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 02b9b57ca1369a..d95245067e2608 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -162,10 +162,6 @@ bytearray_removeprefix(PyByteArrayObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &prefix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&prefix, 'C')) { - _PyArg_BadArgument("removeprefix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytearray_removeprefix_impl(self, &prefix); exit: @@ -202,10 +198,6 @@ bytearray_removesuffix(PyByteArrayObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &suffix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&suffix, 'C')) { - _PyArg_BadArgument("removesuffix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytearray_removesuffix_impl(self, &suffix); exit: @@ -316,17 +308,9 @@ bytearray_maketrans(void *null, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &frm, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&frm, 'C')) { - _PyArg_BadArgument("maketrans", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &to, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&to, 'C')) { - _PyArg_BadArgument("maketrans", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = bytearray_maketrans_impl(&frm, &to); exit: @@ -376,17 +360,9 @@ bytearray_replace(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nar if (PyObject_GetBuffer(args[0], &old, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&old, 'C')) { - _PyArg_BadArgument("replace", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &new, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&new, 'C')) { - _PyArg_BadArgument("replace", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -1285,4 +1261,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=fc2b9ccabe0e6782 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0797a5e03cda2a16 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 8b08f8edb62109..1e45be3e7aefb3 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -141,10 +141,6 @@ bytes_partition(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &sep, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&sep, 'C')) { - _PyArg_BadArgument("partition", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_partition_impl(self, &sep); exit: @@ -184,10 +180,6 @@ bytes_rpartition(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &sep, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&sep, 'C')) { - _PyArg_BadArgument("rpartition", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_rpartition_impl(self, &sep); exit: @@ -503,17 +495,9 @@ bytes_maketrans(void *null, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &frm, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&frm, 'C')) { - _PyArg_BadArgument("maketrans", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &to, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&to, 'C')) { - _PyArg_BadArgument("maketrans", "argument 2", "contiguous buffer", args[1]); - goto exit; - } return_value = bytes_maketrans_impl(&frm, &to); exit: @@ -563,17 +547,9 @@ bytes_replace(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &old, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&old, 'C')) { - _PyArg_BadArgument("replace", "argument 1", "contiguous buffer", args[0]); - goto exit; - } if (PyObject_GetBuffer(args[1], &new, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&new, 'C')) { - _PyArg_BadArgument("replace", "argument 2", "contiguous buffer", args[1]); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -629,10 +605,6 @@ bytes_removeprefix(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &prefix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&prefix, 'C')) { - _PyArg_BadArgument("removeprefix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_removeprefix_impl(self, &prefix); exit: @@ -669,10 +641,6 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &suffix, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&suffix, 'C')) { - _PyArg_BadArgument("removesuffix", "argument", "contiguous buffer", arg); - goto exit; - } return_value = bytes_removesuffix_impl(self, &suffix); exit: @@ -1061,4 +1029,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=da013a7e257f5c6e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a49dbbd78914a6f input=a9049054013a1b77]*/ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 48623451c0d2cd..e3e8e02861d775 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3700,10 +3700,6 @@ _PyExc_FiniTypes(PyInterpreterState *interp) PyStatus _PyExc_InitGlobalObjects(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return _PyStatus_OK(); - } - if (preallocate_memerrors() < 0) { return _PyStatus_NO_MEMORY(); } diff --git a/Objects/frame_layout.md b/Objects/frame_layout.md index 2f95214db56498..b348e85689f507 100644 --- a/Objects/frame_layout.md +++ b/Objects/frame_layout.md @@ -130,3 +130,28 @@ returns. This extra frame is inserted so that `RETURN_VALUE`, `YIELD_VALUE`, and `RETURN_GENERATOR` do not need to check whether the current frame is the entry frame. The shim frame points to a special code object containing the `INTERPRETER_EXIT` instruction which cleans up the shim frame and returns. + + +### The Instruction Pointer + +`_PyInterpreterFrame` has two fields which are used to maintain the instruction +pointer: `instr_ptr` and `return_offset`. + +When a frame is executing, `instr_ptr` points to the instruction currently being +executed. In a suspended frame, it points to the instruction that would execute +if the frame were to resume. After `frame.f_lineno` is set, `instr_ptr` points to +the next instruction to be executed. During a call to a python function, +`instr_ptr` points to the call instruction, because this is what we would expect +to see in an exception traceback. + +The `return_offset` field determines where a `RETURN` should go in the caller, +relative to `instr_ptr`. It is only meaningful to the callee, so it needs to +be set in any instruction that implements a call (to a Python function), +including CALL, SEND and BINARY_SUBSCR_GETITEM, among others. If there is no +callee, then return_offset is meaningless. It is necessary to have a separate +field for the return offset because (1) if we apply this offset to `instr_ptr` +while executing the `RETURN`, this is too early and would lose us information +about the previous instruction which we could need for introspecting and +debugging. (2) `SEND` needs to pass two offsets to the generator: one for +`RETURN` and one for `YIELD`. It uses the `oparg` for one, and the +`return_offset` for the other. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d75444393f3697..698e88483569d1 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -329,6 +329,8 @@ mark_stacks(PyCodeObject *code_obj, int len) switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: { int64_t target_stack; int j = next_i + oparg; @@ -819,7 +821,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } /* Finally set the new lasti and return OK. */ f->f_lineno = 0; - f->f_frame->prev_instr = _PyCode_CODE(code) + best_addr; + f->f_frame->instr_ptr = _PyCode_CODE(code) + best_addr; return 0; } @@ -1077,7 +1079,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; // This frame needs to be "complete", so pretend that the first RESUME ran: - f->f_frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + f->f_frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; assert(!_PyFrame_IsIncomplete(f->f_frame)); Py_DECREF(func); _PyObject_GC_TRACK(f); @@ -1091,7 +1093,7 @@ _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) assert(_PyOpcode_Deopt[opcode] == opcode); int check_oparg = 0; for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame)); - instruction < frame->prev_instr; instruction++) + instruction < frame->instr_ptr; instruction++) { int check_opcode = _PyOpcode_Deopt[instruction->op.code]; check_oparg |= instruction->op.arg; @@ -1133,7 +1135,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) frame->localsplus[offset + i] = Py_NewRef(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); } diff --git a/Objects/genobject.c b/Objects/genobject.c index 491e631f2242e9..14771396619a3a 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -9,6 +9,8 @@ #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -362,8 +364,7 @@ _PyGen_yf(PyGenObject *gen) assert(_PyCode_CODE(_PyGen_GetCode(gen))[0].op.code != SEND); return NULL; } - _Py_CODEUNIT next = frame->prev_instr[1]; - if (!is_resume(&next) || next.op.arg < 2) + if (!is_resume(frame->instr_ptr) || frame->instr_ptr->op.arg < RESUME_AFTER_YIELD_FROM) { /* Not in a yield from */ return NULL; @@ -378,7 +379,6 @@ static PyObject * gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyObject *yf = _PyGen_yf(gen); int err = 0; if (gen->gi_frame_state == FRAME_CREATED) { @@ -388,6 +388,7 @@ gen_close(PyGenObject *gen, PyObject *args) if (gen->gi_frame_state >= FRAME_COMPLETED) { Py_RETURN_NONE; } + PyObject *yf = _PyGen_yf(gen); if (yf) { PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; @@ -398,14 +399,18 @@ gen_close(PyGenObject *gen, PyObject *args) _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; /* It is possible for the previous instruction to not be a * YIELD_VALUE if the debugger has changed the lineno. */ - if (err == 0 && is_yield(frame->prev_instr)) { - assert(is_resume(frame->prev_instr + 1)); - int exception_handler_depth = frame->prev_instr[0].op.code; + assert(_PyOpcode_Caches[YIELD_VALUE] == 0); + assert(_PyOpcode_Caches[INSTRUMENTED_YIELD_VALUE] == 0); + if (err == 0 && is_yield(frame->instr_ptr - 1)) { + _Py_CODEUNIT *yield_instr = frame->instr_ptr - 1; + assert(is_resume(frame->instr_ptr)); + int exception_handler_depth = yield_instr->op.arg; assert(exception_handler_depth > 0); /* We can safely ignore the outermost try block * as it automatically generated to handle * StopIteration. */ if (exception_handler_depth == 1) { + gen->gi_frame_state = FRAME_COMPLETED; Py_RETURN_NONE; } } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3261a14a053dc8..25085693070221 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3499,13 +3499,14 @@ type_new_set_doc(PyTypeObject *type) return 0; } - const char *doc_str = PyUnicode_AsUTF8(doc); + Py_ssize_t doc_size; + const char *doc_str = PyUnicode_AsUTF8AndSize(doc, &doc_size); if (doc_str == NULL) { return -1; } // Silently truncate the docstring if it contains a null byte - Py_ssize_t size = strlen(doc_str) + 1; + Py_ssize_t size = doc_size + 1; char *tp_doc = (char *)PyObject_Malloc(size); if (tp_doc == NULL) { PyErr_NoMemory(); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 33cbc987d43282..80b19567c63d20 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3820,24 +3820,37 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); + if (psize) { + *psize = -1; + } return NULL; } if (PyUnicode_UTF8(unicode) == NULL) { if (unicode_fill_utf8(unicode) == -1) { + if (psize) { + *psize = -1; + } return NULL; } } - if (psize) + if (psize) { *psize = PyUnicode_UTF8_LENGTH(unicode); + } return PyUnicode_UTF8(unicode); } const char * PyUnicode_AsUTF8(PyObject *unicode) { - return PyUnicode_AsUTF8AndSize(unicode, NULL); + Py_ssize_t size; + const char *utf8 = PyUnicode_AsUTF8AndSize(unicode, &size); + if (utf8 != NULL && strlen(utf8) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + return utf8; } /* diff --git a/PC/python3dll.c b/PC/python3dll.c index d12889f44d65b6..7f5d97ae4dc83f 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -661,6 +661,7 @@ EXPORT_FUNC(PyUnicode_AsUCS4Copy) EXPORT_FUNC(PyUnicode_AsUnicodeEscapeString) EXPORT_FUNC(PyUnicode_AsUTF16String) EXPORT_FUNC(PyUnicode_AsUTF32String) +EXPORT_FUNC(PyUnicode_AsUTF8) EXPORT_FUNC(PyUnicode_AsUTF8AndSize) EXPORT_FUNC(PyUnicode_AsUTF8String) EXPORT_FUNC(PyUnicode_AsWideChar) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 0f33c5a76ade9d..ae97c34eed50e0 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -115,6 +115,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 4ba6011d8af5b9..c3ad2564edd686 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -72,6 +72,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 44115da8629e42..232c569a3ddb30 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -920,6 +920,10 @@ break; } + case _SAVE_RETURN_OFFSET: { + break; + } + case _EXIT_TRACE: { break; } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 41f48eba08afc4..04d7ae6eaafbc0 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -142,15 +142,6 @@ check_complexity(PyObject *obj, Py_ssize_t limit) } return limit; } - else if (PyFrozenSet_Check(obj)) { - Py_ssize_t i = 0; - PyObject *item; - Py_hash_t hash; - limit -= PySet_GET_SIZE(obj); - while (limit >= 0 && _PySet_NextEntry(obj, &i, &item, &hash)) { - limit = check_complexity(item, limit); - } - } return limit; } @@ -174,9 +165,8 @@ safe_multiply(PyObject *v, PyObject *w) return NULL; } } - else if (PyLong_Check(v) && (PyTuple_Check(w) || PyFrozenSet_Check(w))) { - Py_ssize_t size = PyTuple_Check(w) ? PyTuple_GET_SIZE(w) : - PySet_GET_SIZE(w); + else if (PyLong_Check(v) && PyTuple_Check(w)) { + Py_ssize_t size = PyTuple_GET_SIZE(w); if (size) { long n = PyLong_AsLong(v); if (n < 0 || n > MAX_COLLECTION_SIZE / size) { @@ -198,8 +188,7 @@ safe_multiply(PyObject *v, PyObject *w) } } else if (PyLong_Check(w) && - (PyTuple_Check(v) || PyFrozenSet_Check(v) || - PyUnicode_Check(v) || PyBytes_Check(v))) + (PyTuple_Check(v) || PyUnicode_Check(v) || PyBytes_Check(v))) { return safe_multiply(w, v); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d7e2ecdd24dcee..2d7b5ba21ea09d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -147,7 +147,7 @@ dummy_func( next_instr--; } else { - if (oparg < 2) { + if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } next_instr[-1].op.code = RESUME_CHECK; @@ -183,9 +183,9 @@ dummy_func( tstate, oparg > 0, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); - if (frame->prev_instr != next_instr-1) { + if (frame->instr_ptr != next_instr-1) { /* Instrumentation has jumped */ - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } @@ -657,7 +657,8 @@ dummy_func( new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -790,10 +791,9 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -803,7 +803,6 @@ dummy_func( } macro(RETURN_VALUE) = - _SAVE_CURRENT_IP + // Sets frame->prev_instr _POP_FRAME; inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { @@ -820,14 +819,13 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } macro(RETURN_CONST) = LOAD_CONST + - _SAVE_CURRENT_IP + // Sets frame->prev_instr _POP_FRAME; inst(INSTRUMENTED_RETURN_CONST, (--)) { @@ -845,8 +843,8 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -982,7 +980,8 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -1020,13 +1019,15 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1041,6 +1042,9 @@ dummy_func( frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1050,6 +1054,7 @@ dummy_func( // or throw() call. assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ assert(frame != &entry_frame); + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1060,6 +1065,9 @@ dummy_func( frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1073,7 +1081,7 @@ dummy_func( if (oparg) { PyObject *lasti = values[0]; if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2011,7 +2019,8 @@ dummy_func( STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2038,7 +2047,8 @@ dummy_func( new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2306,13 +2316,14 @@ dummy_func( _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); JUMPBY(1-original_oparg); - frame->prev_instr = next_instr - 1; + frame->instr_ptr = next_instr; Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { frame = tstate->current_frame; goto resume_with_error; } + next_instr = frame->instr_ptr; goto resume_frame; } @@ -2478,7 +2489,7 @@ dummy_func( } inst(INSTRUMENTED_FOR_ITER, ( -- )) { - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2674,7 +2685,8 @@ dummy_func( SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -2985,7 +2997,8 @@ dummy_func( goto error; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -3072,7 +3085,6 @@ dummy_func( op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -3080,7 +3092,7 @@ dummy_func( frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -3097,7 +3109,7 @@ dummy_func( _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + - _SAVE_CURRENT_IP + // Sets frame->prev_instr + _SAVE_RETURN_OFFSET + _PUSH_FRAME; macro(CALL_PY_EXACT_ARGS) = @@ -3106,7 +3118,7 @@ dummy_func( _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + - _SAVE_CURRENT_IP + // Sets frame->prev_instr + _SAVE_RETURN_OFFSET + _PUSH_FRAME; inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { @@ -3140,7 +3152,8 @@ dummy_func( // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3205,7 +3218,7 @@ dummy_func( Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; @@ -3217,8 +3230,8 @@ dummy_func( init_frame->localsplus[i+1] = args[i]; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); /* Link frames */ @@ -3587,7 +3600,8 @@ dummy_func( if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -3683,7 +3697,8 @@ dummy_func( if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); @@ -3746,6 +3761,7 @@ dummy_func( assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; @@ -3756,6 +3772,7 @@ dummy_func( _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -3838,18 +3855,20 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT *here = frame->instr_ptr; + INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { + _Py_CODEUNIT *here = frame->instr_ptr; CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3861,7 +3880,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3872,7 +3891,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -3890,7 +3909,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { @@ -3945,16 +3964,20 @@ dummy_func( op(_SET_IP, (--)) { TIER_TWO_ONLY - frame->prev_instr = ip_offset + oparg; + frame->instr_ptr = ip_offset + oparg; } - op(_SAVE_CURRENT_IP, (--)) { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + op(_SAVE_RETURN_OFFSET, (--)) { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } op(_EXIT_TRACE, (--)) { - frame->prev_instr--; // Back up to just before destination + TIER_TWO_ONLY _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/ceval.c b/Python/ceval.c index ac40425263931f..e3a7c5f38403a7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -640,8 +640,10 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, - { .op.code = RESUME, .op.arg = 0 } + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ + { .op.code = RESUME, .op.arg = RESUME_AT_FUNC_START } }; extern const struct _PyCode_DEF(8) _Py_InitCleanup; @@ -698,7 +700,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.f_builtins = (PyObject*)0xaaa4; #endif entry_frame.f_executable = Py_None; - entry_frame.prev_instr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stacktop = 0; entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; @@ -722,7 +724,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, * we need to update instrumentation */ _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - monitor_throw(tstate, frame, frame->prev_instr); + monitor_throw(tstate, frame, frame->instr_ptr); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -733,19 +735,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; PyObject **stack_pointer; -/* Sets the above local variables from the frame */ -#define SET_LOCALS_FROM_FRAME() \ - /* Jump back to the last instruction executed... */ \ - next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_GetStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } + next_instr = frame->instr_ptr; resume_frame: - SET_LOCALS_FROM_FRAME(); + stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); @@ -774,7 +772,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->prev_instr before it is updated, + * because it needs to capture frame->instr_ptr before it is updated, * as happens in the standard instruction prologue. */ #if USE_COMPUTED_GOTOS @@ -783,8 +781,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int case INSTRUMENTED_LINE: #endif { - _Py_CODEUNIT *prev = frame->prev_instr; - _Py_CODEUNIT *here = frame->prev_instr = next_instr; + _Py_CODEUNIT *prev = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here, prev); @@ -793,7 +791,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = here+1; goto error; } - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; if (next_instr != here) { DISPATCH(); } @@ -908,7 +906,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); PUSH(exc); - JUMPTO(handler); + next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { goto exception_unwind; } @@ -939,7 +938,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } resume_with_error: - SET_LOCALS_FROM_FRAME(); + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index bd28126b0b7dfb..544e8ef8fa8c0a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -62,13 +62,16 @@ #ifdef Py_STATS #define INSTRUCTION_START(op) \ do { \ - frame->prev_instr = next_instr++; \ + frame->instr_ptr = next_instr++; \ OPCODE_EXE_INC(op); \ if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else -#define INSTRUCTION_START(op) (frame->prev_instr = next_instr++) +#define INSTRUCTION_START(op) \ + do { \ + frame->instr_ptr = next_instr++; \ + } while(0) #endif #if USE_COMPUTED_GOTOS @@ -107,7 +110,6 @@ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ @@ -146,7 +148,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) -#define JUMPTO(x) (next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + (x)) /* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is * for advancing to the next instruction, taking into account cache entries @@ -381,8 +382,9 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { #if TIER_ONE -#define LOAD_IP() \ -do { next_instr = frame->prev_instr+1; } while (0) +#define LOAD_IP(OFFSET) do { \ + next_instr = frame->instr_ptr + (OFFSET); \ + } while (0) #define STORE_SP() \ _PyFrame_SetStackPointer(frame, stack_pointer) @@ -395,7 +397,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); #if TIER_TWO -#define LOAD_IP() \ +#define LOAD_IP(UNUSED) \ do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0) #define STORE_SP() \ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 3ecd2acb1cd286..5edeaef656ef62 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -605,10 +605,6 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (PyObject_GetBuffer(args[1], &source, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&source, 'C')) { - _PyArg_BadArgument("source_hash", "argument 'source'", "contiguous buffer", args[1]); - goto exit; - } return_value = _imp_source_hash_impl(module, key, &source); exit: @@ -627,4 +623,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=c37ad1bf06f9e947 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dbd63707bd40b07c input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index ecde4f6482d30e..e6b0f1999a41c5 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -143,10 +143,6 @@ marshal_loads(PyObject *module, PyObject *arg) if (PyObject_GetBuffer(arg, &bytes, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&bytes, 'C')) { - _PyArg_BadArgument("loads", "argument", "contiguous buffer", arg); - goto exit; - } return_value = marshal_loads_impl(module, &bytes); exit: @@ -157,4 +153,4 @@ marshal_loads(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=99ba446b1a75a269 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=92d2d47aac9128ee input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 1d9ae626677310..3ff64182ba2dff 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1383,7 +1383,7 @@ compiler_enter_scope(struct compiler *c, identifier name, else { RETURN_IF_ERROR(compiler_set_qualname(c)); } - ADDOP_I(c, loc, RESUME, 0); + ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = -1; @@ -1552,7 +1552,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); ADDOP_I(c, loc, YIELD_VALUE, 0); ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_I(c, loc, RESUME, await ? 3 : 2); + ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM); ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send); USE_LABEL(c, fail); @@ -4161,7 +4161,7 @@ addop_yield(struct compiler *c, location loc) { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); } ADDOP_I(c, loc, YIELD_VALUE, 0); - ADDOP_I(c, loc, RESUME, 1); + ADDOP_I(c, loc, RESUME, RESUME_AFTER_YIELD); return SUCCESS; } diff --git a/Python/executor.c b/Python/executor.c index 1630f018626449..bfa7f7e1c3d84e 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -62,7 +62,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), _PyFrame_GetCode(frame)->co_firstlineno, - 2 * (long)(frame->prev_instr + 1 - + 2 * (long)(frame->instr_ptr - (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); PyThreadState *tstate = _PyThreadState_GET(); @@ -131,6 +131,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject // The caller recovers the frame from tstate->current_frame. DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return NULL; @@ -140,7 +141,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject // This presumes nothing was popped from the stack (nor pushed). DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->prev_instr--; // Back up to just before destination + frame->return_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return frame; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 119e77b9369b13..9f37cd75b47efb 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -692,10 +692,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -2587,7 +2586,6 @@ STACK_SHRINK(1); // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -2595,7 +2593,7 @@ frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -3271,12 +3269,22 @@ case _SET_IP: { TIER_TWO_ONLY - frame->prev_instr = ip_offset + oparg; + frame->instr_ptr = ip_offset + oparg; + break; + } + + case _SAVE_RETURN_OFFSET: { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif break; } case _EXIT_TRACE: { - frame->prev_instr--; // Back up to just before destination + TIER_TWO_ONLY _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/frame.c b/Python/frame.c index b483903fdf3018..2865b2eab603c2 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -90,7 +90,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: PyCodeObject *code = _PyFrame_GetCode(frame); - frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; } assert(!_PyFrame_IsIncomplete(frame)); assert(f->f_back == NULL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6fbe80fe03a128..d5e0d849efe9d2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -23,7 +23,7 @@ next_instr--; } else { - if (oparg < 2) { + if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } next_instr[-1].op.code = RESUME_CHECK; @@ -61,9 +61,9 @@ tstate, oparg > 0, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; - if (frame->prev_instr != next_instr-1) { + if (frame->instr_ptr != next_instr-1) { /* Instrumentation has jumped */ - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } @@ -804,7 +804,8 @@ new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -987,36 +988,27 @@ TARGET(RETURN_VALUE) { PyObject *retval; - // _SAVE_CURRENT_IP - { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; - } - // _POP_FRAME retval = stack_pointer[-1]; STACK_SHRINK(1); - { - assert(EMPTY()); - #if TIER_ONE - assert(frame != &entry_frame); - #endif - STORE_SP(); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + assert(EMPTY()); + #if TIER_ONE + assert(frame != &entry_frame); + #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; } +#endif DISPATCH(); } @@ -1036,8 +1028,8 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -1049,11 +1041,6 @@ value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); } - // _SAVE_CURRENT_IP - { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; - } // _POP_FRAME retval = value; { @@ -1067,10 +1054,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -1096,8 +1082,8 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -1252,7 +1238,8 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -1297,7 +1284,8 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1306,6 +1294,7 @@ retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1320,6 +1309,9 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1331,6 +1323,7 @@ // or throw() call. assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ assert(frame != &entry_frame); + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1341,6 +1334,9 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1362,7 +1358,7 @@ if (oparg) { PyObject *lasti = values[0]; if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2591,7 +2587,8 @@ STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2623,7 +2620,8 @@ new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2999,13 +2997,14 @@ _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); JUMPBY(1-original_oparg); - frame->prev_instr = next_instr - 1; + frame->instr_ptr = next_instr; Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { frame = tstate->current_frame; goto resume_with_error; } + next_instr = frame->instr_ptr; goto resume_frame; } @@ -3272,7 +3271,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3440,7 +3439,8 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -3828,7 +3828,8 @@ goto error; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -3928,11 +3929,15 @@ new_frame->localsplus[i] = args[i]; } } - // _SAVE_CURRENT_IP + // _SAVE_RETURN_OFFSET next_instr += 3; { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } // _PUSH_FRAME STACK_SHRINK(oparg); @@ -3940,7 +3945,6 @@ { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -3948,7 +3952,7 @@ frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -4001,11 +4005,15 @@ new_frame->localsplus[i] = args[i]; } } - // _SAVE_CURRENT_IP + // _SAVE_RETURN_OFFSET next_instr += 3; { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } // _PUSH_FRAME STACK_SHRINK(oparg); @@ -4013,7 +4021,6 @@ { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -4021,7 +4028,7 @@ frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -4070,7 +4077,8 @@ // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -4177,7 +4185,7 @@ Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; @@ -4189,8 +4197,8 @@ init_frame->localsplus[i+1] = args[i]; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); /* Link frames */ @@ -4698,7 +4706,8 @@ if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -4806,7 +4815,8 @@ if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); @@ -4887,6 +4897,7 @@ assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; @@ -4897,6 +4908,7 @@ _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -5034,20 +5046,22 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT *here = frame->instr_ptr; + INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *here = frame->instr_ptr; CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -5061,7 +5075,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -5074,7 +5088,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -5094,7 +5108,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { diff --git a/Python/getargs.c b/Python/getargs.c index 80d2540a694d7a..5a12ca8def74fa 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1187,17 +1187,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, arg, msgbuf, bufsize); format++; - /* Caller is interested in Py_buffer, and the object - supports it directly. */ + /* Caller is interested in Py_buffer, and the object supports it + directly. The request implicitly asks for PyBUF_SIMPLE, so the + result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } - if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) { - PyBuffer_Release((Py_buffer*)p); - return converterr("contiguous buffer", arg, msgbuf, bufsize); - } + assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( "(cleanup problem)", @@ -1242,15 +1240,12 @@ convertbuffer(PyObject *arg, const void **p, const char **errmsg) static int getbuffer(PyObject *arg, Py_buffer *view, const char **errmsg) { + /* PyBUF_SIMPLE implies C-contiguous */ if (PyObject_GetBuffer(arg, view, PyBUF_SIMPLE) != 0) { *errmsg = "bytes-like object"; return -1; } - if (!PyBuffer_IsContiguous(view, 'C')) { - PyBuffer_Release(view); - *errmsg = "contiguous buffer"; - return -1; - } + assert(PyBuffer_IsContiguous(view, 'C')); return 0; } diff --git a/Python/import.c b/Python/import.c index cafdd834502224..9aee7aa0e4d1f1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,7 +1,6 @@ /* Module definition and import implementation */ #include "Python.h" -#include "pycore_dict.h" // _PyDict_Pop() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() diff --git a/Python/instrumentation.c b/Python/instrumentation.c index eee1908e503e43..5fd65d53c503f8 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1073,7 +1073,7 @@ _Py_call_instrumentation_jump( { assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); - assert(frame->prev_instr == instr); + assert(frame->instr_ptr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1086,9 +1086,9 @@ _Py_call_instrumentation_jump( if (err) { return NULL; } - if (frame->prev_instr != instr) { + if (frame->instr_ptr != instr) { /* The callback has caused a jump (by setting the line number) */ - return frame->prev_instr; + return frame->instr_ptr; } return target; } @@ -1138,7 +1138,6 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { - assert(frame->prev_instr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); @@ -1153,6 +1152,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); assert(line >= 0); + assert(prev != NULL); int prev_index = (int)(prev - _PyCode_CODE(code)); int prev_line = _Py_Instrumentation_GetLine(code, prev_index); if (prev_line == line) { @@ -1582,6 +1582,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } + _Py_Executors_InvalidateDependency(interp, code); int code_len = (int)Py_SIZE(code); /* code->_co_firsttraceable >= code_len indicates * that no instrumentation can be inserted. @@ -1803,6 +1804,7 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) return -1; } set_global_version(interp, new_version); + _Py_Executors_InvalidateAll(interp); return instrument_all_executing_code_objects(interp); } @@ -1832,6 +1834,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent /* Force instrumentation update */ code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } + _Py_Executors_InvalidateDependency(interp, code); if (_Py_Instrument(code, interp)) { return -1; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 955ac812177ac4..6402287a412a35 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -220,10 +220,22 @@ typedef struct { static void counter_dealloc(_PyCounterExecutorObject *self) { + _Py_ExecutorClear((_PyExecutorObject *)self); Py_DECREF(self->optimizer); PyObject_Free(self); } +static PyObject * +is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid); +} + +static PyMethodDef executor_methods[] = { + { "is_valid", is_valid, METH_NOARGS, NULL }, + { NULL, NULL }, +}; + static PyTypeObject CounterExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "counting_executor", @@ -231,6 +243,7 @@ static PyTypeObject CounterExecutor_Type = { .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)counter_dealloc, + .tp_methods = executor_methods, }; static _PyInterpreterFrame * @@ -238,7 +251,7 @@ counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject ** { ((_PyCounterExecutorObject *)self)->optimizer->count++; _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = ((_PyCounterExecutorObject *)self)->next_instr - 1; + frame->instr_ptr = ((_PyCounterExecutorObject *)self)->next_instr; Py_DECREF(self); return frame; } @@ -261,6 +274,9 @@ counter_optimize( executor->optimizer = (_PyCounterOptimizerObject *)self; executor->next_instr = instr; *exec_ptr = (_PyExecutorObject *)executor; + _PyBloomFilter empty; + _Py_BloomFilter_Init(&empty); + _Py_ExecutorInit((_PyExecutorObject *)executor, &empty); return 1; } @@ -270,7 +286,7 @@ counter_get_counter(PyObject *self, PyObject *args) return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); } -static PyMethodDef counter_methods[] = { +static PyMethodDef counter_optimizer_methods[] = { { "get_count", counter_get_counter, METH_NOARGS, NULL }, { NULL, NULL }, }; @@ -281,13 +297,15 @@ static PyTypeObject CounterOptimizer_Type = { .tp_basicsize = sizeof(_PyCounterOptimizerObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_methods = counter_methods, + .tp_methods = counter_optimizer_methods, .tp_dealloc = (destructor)PyObject_Del, }; PyObject * PyUnstable_Optimizer_NewCounter(void) { + PyType_Ready(&CounterExecutor_Type); + PyType_Ready(&CounterOptimizer_Type); _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&CounterOptimizer_Type); if (opt == NULL) { return NULL; @@ -303,6 +321,7 @@ PyUnstable_Optimizer_NewCounter(void) static void uop_dealloc(_PyUOpExecutorObject *self) { + _Py_ExecutorClear((_PyExecutorObject *)self); PyObject_Free(self); } @@ -364,6 +383,7 @@ static PyTypeObject UOpExecutor_Type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, + .tp_methods = executor_methods, }; static int @@ -399,9 +419,11 @@ translate_bytecode_to_trace( PyCodeObject *code, _Py_CODEUNIT *instr, _PyUOpInstruction *trace, - int buffer_size) + int buffer_size, + _PyBloomFilter *dependencies) { PyCodeObject *initial_code = code; + _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; int max_length = buffer_size; @@ -679,9 +701,9 @@ translate_bytecode_to_trace( case OPARG_BOTTOM: // Second half of super-instr oparg = orig_oparg & 0xF; break; - case OPARG_SET_IP: // op==_SET_IP; oparg=next instr - oparg = INSTR_IP(instr + offset, code); - uop = _SET_IP; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); break; default: @@ -735,6 +757,7 @@ translate_bytecode_to_trace( // Increment IP to the return address instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); + _Py_BloomFilter_Add(dependencies, new_code); code = new_code; instr = _PyCode_CODE(code); DPRINTF(2, @@ -827,11 +850,7 @@ remove_unneeded_uops(_PyUOpInstruction *trace, int trace_length) bool need_ip = true; for (int pc = 0; pc < trace_length; pc++) { int opcode = trace[pc].opcode; - if (opcode == _SAVE_CURRENT_IP) { - // Special case: never remove preceding _SET_IP - last_set_ip = -1; - } - else if (opcode == _SET_IP) { + if (opcode == _SET_IP) { if (!need_ip && last_set_ip >= 0) { trace[last_set_ip].opcode = NOP; } @@ -843,8 +862,8 @@ remove_unneeded_uops(_PyUOpInstruction *trace, int trace_length) break; } else { - // If opcode has ERROR or DEOPT, set need_up to true - if (_PyOpcode_opcode_metadata[opcode].flags & (HAS_ERROR_FLAG | HAS_DEOPT_FLAG)) { + // If opcode has ERROR or DEOPT, set need_ip to true + if (_PyOpcode_opcode_metadata[opcode].flags & (HAS_ERROR_FLAG | HAS_DEOPT_FLAG) || opcode == _PUSH_FRAME) { need_ip = true; } } @@ -895,8 +914,10 @@ uop_optimize( _PyExecutorObject **exec_ptr, int curr_stackentries) { + _PyBloomFilter dependencies; + _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; - int trace_length = translate_bytecode_to_trace(code, instr, trace, _Py_UOP_MAX_TRACE_LENGTH); + int trace_length = translate_bytecode_to_trace(code, instr, trace, _Py_UOP_MAX_TRACE_LENGTH, &dependencies); if (trace_length <= 0) { // Error or nothing translated return trace_length; @@ -915,6 +936,7 @@ uop_optimize( OPT_HIST(trace_length, optimized_trace_length_hist); executor->base.execute = _PyUopExecute; memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction)); + _Py_ExecutorInit((_PyExecutorObject *)executor, &dependencies); *exec_ptr = (_PyExecutorObject *)executor; return 1; } @@ -936,6 +958,8 @@ static PyTypeObject UOpOptimizer_Type = { PyObject * PyUnstable_Optimizer_NewUOpOptimizer(void) { + PyType_Ready(&UOpExecutor_Type); + PyType_Ready(&UOpOptimizer_Type); _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &UOpOptimizer_Type); if (opt == NULL) { return NULL; @@ -947,3 +971,206 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) opt->backedge_threshold = 16 << OPTIMIZER_BITS_IN_COUNTER; return (PyObject *)opt; } + + +/***************************************** + * Executor management + ****************************************/ + +/* We use a bloomfilter with k = 6, m = 256 + * The choice of k and the following constants + * could do with a more rigourous analysis, + * but here is a simple analysis: + * + * We want to keep the false positive rate low. + * For n = 5 (a trace depends on 5 objects), + * we expect 30 bits set, giving a false positive + * rate of (30/256)**6 == 2.5e-6 which is plenty + * good enough. + * + * However with n = 10 we expect 60 bits set (worst case), + * giving a false positive of (60/256)**6 == 0.0001 + * + * We choose k = 6, rather than a higher number as + * it means the false positive rate grows slower for high n. + * + * n = 5, k = 6 => fp = 2.6e-6 + * n = 5, k = 8 => fp = 3.5e-7 + * n = 10, k = 6 => fp = 1.6e-4 + * n = 10, k = 8 => fp = 0.9e-4 + * n = 15, k = 6 => fp = 0.18% + * n = 15, k = 8 => fp = 0.23% + * n = 20, k = 6 => fp = 1.1% + * n = 20, k = 8 => fp = 2.3% + * + * The above analysis assumes perfect hash functions, + * but those don't exist, so the real false positive + * rates may be worse. + */ + +#define K 6 + +#define SEED 20221211 + +/* TO DO -- Use more modern hash functions with better distribution of bits */ +static uint64_t +address_to_hash(void *ptr) { + assert(ptr != NULL); + uint64_t uhash = SEED; + uintptr_t addr = (uintptr_t)ptr; + for (int i = 0; i < SIZEOF_VOID_P; i++) { + uhash ^= addr & 255; + uhash *= (uint64_t)_PyHASH_MULTIPLIER; + addr >>= 8; + } + return uhash; +} + +void +_Py_BloomFilter_Init(_PyBloomFilter *bloom) +{ + for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + bloom->bits[i] = 0; + } +} + +/* We want K hash functions that each set 1 bit. + * A hash function that sets 1 bit in M bits can be trivially + * derived from a log2(M) bit hash function. + * So we extract 8 (log2(256)) bits at a time from + * the 64bit hash. */ +void +_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) +{ + uint64_t hash = address_to_hash(ptr); + assert(K <= 8); + for (int i = 0; i < K; i++) { + uint8_t bits = hash & 255; + bloom->bits[bits >> 5] |= (1 << (bits&31)); + hash >>= 8; + } +} + +static bool +bloom_filter_may_contain(_PyBloomFilter *bloom, _PyBloomFilter *hashes) +{ + for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { + return false; + } + } + return true; +} + +static void +link_executor(_PyExecutorObject *executor) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyExecutorLinkListNode *links = &executor->vm_data.links; + _PyExecutorObject *head = interp->executor_list_head; + if (head == NULL) { + interp->executor_list_head = executor; + links->previous = NULL; + links->next = NULL; + } + else { + _PyExecutorObject *next = head->vm_data.links.next; + links->previous = head; + links->next = next; + if (next != NULL) { + next->vm_data.links.previous = executor; + } + head->vm_data.links.next = executor; + } + executor->vm_data.linked = true; + /* executor_list_head must be first in list */ + assert(interp->executor_list_head->vm_data.links.previous == NULL); +} + +static void +unlink_executor(_PyExecutorObject *executor) +{ + if (!executor->vm_data.linked) { + return; + } + _PyExecutorLinkListNode *links = &executor->vm_data.links; + _PyExecutorObject *next = links->next; + _PyExecutorObject *prev = links->previous; + if (next != NULL) { + next->vm_data.links.previous = prev; + } + if (prev != NULL) { + prev->vm_data.links.next = next; + } + else { + // prev == NULL implies that executor is the list head + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(interp->executor_list_head == executor); + interp->executor_list_head = next; + } + executor->vm_data.linked = false; +} + +/* This must be called by optimizers before using the executor */ +void +_Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) +{ + executor->vm_data.valid = true; + for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + executor->vm_data.bloom.bits[i] = dependency_set->bits[i]; + } + link_executor(executor); +} + +/* This must be called by executors during dealloc */ +void +_Py_ExecutorClear(_PyExecutorObject *executor) +{ + unlink_executor(executor); +} + +void +_Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) +{ + assert(executor->vm_data.valid = true); + _Py_BloomFilter_Add(&executor->vm_data.bloom, obj); +} + +/* Invalidate all executors that depend on `obj` + * May cause other executors to be invalidated as well + */ +void +_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { + exec->vm_data.valid = false; + unlink_executor(exec); + } + exec = next; + } +} + +/* Invalidate all executors */ +void +_Py_Executors_InvalidateAll(PyInterpreterState *interp) +{ + /* Walk the list of executors */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + exec->vm_data.links.next = NULL; + exec->vm_data.links.previous = NULL; + exec->vm_data.valid = false; + exec->vm_data.linked = false; + exec = next; + } + interp->executor_list_head = NULL; +} diff --git a/Python/pystate.c b/Python/pystate.c index 2e6f07e60033ed..c44a28ca6d3ac8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -713,6 +713,7 @@ init_interpreter(PyInterpreterState *interp, interp->optimizer_backedge_threshold = _PyOptimizer_Default.backedge_threshold; interp->optimizer_resume_threshold = _PyOptimizer_Default.backedge_threshold; interp->next_func_version = 1; + interp->executor_list_head = NULL; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b915c063d0b456..79aeee1eb6f6b7 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -40,14 +40,17 @@ /* Forward */ static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, - PyCompilerFlags *, PyArena *, PyObject*); + PyCompilerFlags *, PyArena *, PyObject*, int); static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *, PyCompilerFlags *); static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *); static PyObject* pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags); - +static PyObject * +_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source); int _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, @@ -281,7 +284,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, } PyObject *main_dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src); + PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src, 1); _PyArena_Free(arena); Py_DECREF(main_module); if (res == NULL) { @@ -499,16 +502,25 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, int -PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) -{ +_PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags) { PyObject *main_module = PyImport_AddModuleRef("__main__"); if (main_module == NULL) { return -1; } PyObject *dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = PyRun_StringFlags(command, Py_file_input, - dict, dict, flags); + PyObject *res = NULL; + if (name == NULL) { + res = PyRun_StringFlags(command, Py_file_input, dict, dict, flags); + } else { + PyObject* the_name = PyUnicode_FromString(name); + if (!the_name) { + PyErr_Print(); + return -1; + } + res = _PyRun_StringFlagsWithName(command, the_name, Py_file_input, dict, dict, flags, 0); + Py_DECREF(the_name); + } Py_DECREF(main_module); if (res == NULL) { PyErr_Print(); @@ -519,6 +531,12 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return 0; } +int +PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) +{ + return _PyRun_SimpleStringFlagsWithName(command, NULL, flags); +} + int _Py_HandleSystemExit(int *exitcode_p) { @@ -1131,9 +1149,10 @@ void PyErr_DisplayException(PyObject *exc) PyErr_Display(NULL, exc, NULL); } -PyObject * -PyRun_StringFlags(const char *str, int start, PyObject *globals, - PyObject *locals, PyCompilerFlags *flags) +static PyObject * +_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source) { PyObject *ret = NULL; mod_ty mod; @@ -1143,17 +1162,36 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, if (arena == NULL) return NULL; + PyObject* source = NULL; _Py_DECLARE_STR(anon_string, ""); - mod = _PyParser_ASTFromString( - str, &_Py_STR(anon_string), start, flags, arena); - if (mod != NULL) - ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena, NULL); + if (name) { + source = PyUnicode_FromString(str); + if (!source) { + PyErr_Clear(); + } + } else { + name = &_Py_STR(anon_string); + } + + mod = _PyParser_ASTFromString(str, name, start, flags, arena); + + if (mod != NULL) { + ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); + } + Py_XDECREF(source); _PyArena_Free(arena); return ret; } +PyObject * +PyRun_StringFlags(const char *str, int start, PyObject *globals, + PyObject *locals, PyCompilerFlags *flags) { + + return _PyRun_StringFlagsWithName(str, NULL, start, globals, locals, flags, 0); +} + static PyObject * pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) @@ -1173,7 +1211,7 @@ pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *ret; if (mod != NULL) { - ret = run_mod(mod, filename, globals, locals, flags, arena, NULL); + ret = run_mod(mod, filename, globals, locals, flags, arena, NULL, 0); } else { ret = NULL; @@ -1261,15 +1299,19 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py static PyObject * run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src) + PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src, + int generate_new_source) { PyThreadState *tstate = _PyThreadState_GET(); PyObject* interactive_filename = filename; if (interactive_src) { PyInterpreterState *interp = tstate->interp; - interactive_filename = PyUnicode_FromFormat( - "", interp->_interactive_src_count++ - ); + if (generate_new_source) { + interactive_filename = PyUnicode_FromFormat( + "%U-%d", filename, interp->_interactive_src_count++); + } else { + Py_INCREF(interactive_filename); + } if (interactive_filename == NULL) { return NULL; } @@ -1277,7 +1319,9 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); if (co == NULL) { - Py_DECREF(interactive_filename); + if (interactive_src) { + Py_DECREF(interactive_filename); + } return NULL; } diff --git a/Python/specialize.c b/Python/specialize.c index 49633b103b3815..07fd93d29b09dc 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -10,6 +10,7 @@ #include "pycore_moduleobject.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() @@ -134,7 +135,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats) fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); for (int i = 0; i < 256; i++) { - if (_PyOpcode_Caches[i]) { + if (_PyOpcode_Caches[i] && i != JUMP_BACKWARD) { fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); } PRINT_STAT(i, specialization.success); @@ -2524,7 +2525,7 @@ static const PyBytesObject no_location = { }; const struct _PyCode_DEF(8) _Py_InitCleanup = { - _PyVarObject_HEAD_INIT(&PyCode_Type, 4) + _PyVarObject_HEAD_INIT(&PyCode_Type, 3) .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), @@ -2539,9 +2540,8 @@ const struct _PyCode_DEF(8) _Py_InitCleanup = { .co_stacksize = 2, .co_framesize = 2 + FRAME_SPECIALS_SIZE, .co_code_adaptive = { - NOP, 0, EXIT_INIT_CHECK, 0, RETURN_VALUE, 0, - RESUME, 0, + RESUME, RESUME_AT_FUNC_START, } }; diff --git a/Python/symtable.c b/Python/symtable.c index 75ea9e902f4381..da7fec0ee7cf0c 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1813,14 +1813,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case Try_kind: VISIT_SEQ(st, stmt, s->v.Try.body); - VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case TryStar_kind: VISIT_SEQ(st, stmt, s->v.TryStar.body); - VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); + VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); break; case Assert_kind: diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 66c77cdaf17ddf..d436886fe73d99 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -530,7 +530,7 @@ Modules/_testcapimodule.c make_exception_with_doc kwlist - Modules/_testcapimodule.c raise_SIGINT_then_send_None PyId_send - Modules/_testcapimodule.c slot_tp_del PyId___tp_del__ - Modules/_testcapimodule.c test_capsule buffer - -Modules/_testcapimodule.c test_empty_argparse kwlist - +Modules/_testcapimodule.c getargs_empty kwlist - Modules/_testcapimodule.c test_structmembers_new keywords - Modules/_testcapimodule.c getargs_s_hash_int keywords - Modules/_testcapimodule.c - g_dict_watch_events - diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index b2fa0205bea342..53f715db6f24a5 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -372,8 +372,8 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: case Instruction() as instr: part, offset = self.analyze_instruction(instr, offset) parts.append(part) - if instr.name != "_SET_IP": - # _SET_IP in a macro is a no-op in Tier 1 + if instr.name != "_SAVE_RETURN_OFFSET": + # _SAVE_RETURN_OFFSET's oparg does not transfer flags.add(instr.instr_flags) case _: assert_never(component) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 04d617ba9f5173..ef96ed3748b056 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -67,7 +67,7 @@ "OPARG_CACHE_4": 4, "OPARG_TOP": 5, "OPARG_BOTTOM": 6, - "OPARG_SET_IP": 7, + "OPARG_SAVE_RETURN_OFFSET": 7, } INSTR_FMT_PREFIX = "INSTR_FMT_" @@ -658,7 +658,7 @@ def write_macro_expansions( for part in parts: if isinstance(part, Component): # All component instructions must be viable uops - if not part.instr.is_viable_uop() and part.instr.name != "_SAVE_CURRENT_IP": + if not part.instr.is_viable_uop(): # This note just reminds us about macros that cannot # be expanded to Tier 2 uops. It is not an error. # It is sometimes emitted for macros that have a @@ -671,8 +671,8 @@ def write_macro_expansions( ) return if not part.active_caches: - if part.instr.name == "_SAVE_CURRENT_IP": - size, offset = OPARG_SIZES["OPARG_SET_IP"], cache_offset - 1 + if part.instr.name == "_SAVE_RETURN_OFFSET": + size, offset = OPARG_SIZES["OPARG_SAVE_RETURN_OFFSET"], cache_offset else: size, offset = OPARG_SIZES["OPARG_FULL"], 0 else: diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index bba2db8b059da8..69a6c10d6199c8 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -448,7 +448,7 @@ def write_components( ), f"Expected {mgr.instr.name!r} to be the last uop" assert_no_pokes(managers) - if mgr.instr.name == "_SAVE_CURRENT_IP": + if mgr.instr.name == "_SAVE_RETURN_OFFSET": next_instr_is_set = True if cache_offset: out.emit(f"next_instr += {cache_offset};") diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5dd7900851550a..5f94b90ae09bd0 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1432,67 +1432,64 @@ def parser_body( deprecated_keywords[i] = p has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) - if vararg == NO_VARARG: - # FIXME: refactor the code to not declare args_declaration - # if limited_capi is true - if not limited_capi: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywords()') - args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only - ) - nargs = "nargs" - else: - if not limited_capi: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywordsWithVararg()') - args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only, - vararg - ) - nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if limited_capi: parser_code = None fastcall = False - - elif fastcall: - flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS - argname_fmt = 'args[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" - args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); - if (!args) {{ - goto exit; - }} - """ % args_declaration, indent=4)] else: - # positional-or-keyword arguments - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - argname_fmt = 'fastargs[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - declarations += "\nPyObject * const *fastargs;" - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" - fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); - if (!fastargs) {{ - goto exit; - }} - """ % args_declaration, indent=4)] + if vararg == NO_VARARG: + clinic.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywords()') + args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only + ) + nargs = "nargs" + else: + clinic.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywordsWithVararg()') + args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only, + vararg + ) + nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" + + if fastcall: + flags = "METH_FASTCALL|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS + argname_fmt = 'args[%d]' + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) + declarations += "\nPyObject *argsbuf[%s];" % len(converters) + if has_optional_kw: + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) + parser_code = [normalize_snippet(""" + args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); + if (!args) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + else: + # positional-or-keyword arguments + flags = "METH_VARARGS|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + argname_fmt = 'fastargs[%d]' + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) + declarations += "\nPyObject *argsbuf[%s];" % len(converters) + declarations += "\nPyObject * const *fastargs;" + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if has_optional_kw: + declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) + parser_code = [normalize_snippet(""" + fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); + if (!fastargs) {{ + goto exit; + }} + """ % args_declaration, indent=4)] if requires_defining_class: flags = 'METH_METHOD|' + flags @@ -4608,15 +4605,12 @@ def cleanup(self) -> str: return "".join(["if (", name, ".obj) {\n PyBuffer_Release(&", name, ");\n}\n"]) def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + # PyBUF_SIMPLE guarantees that the format units of the buffers are C-contiguous. if self.format_unit == 'y*': return self.format_code(""" if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ goto exit; }}}} - if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - {bad_argument} - goto exit; - }}}} """, argname=argname, bad_argument=self.bad_argument(displayname, 'contiguous buffer', limited_capi=limited_capi), @@ -4635,10 +4629,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st if (PyObject_GetBuffer({argname}, &{paramname}, PyBUF_SIMPLE) != 0) {{{{ goto exit; }}}} - if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - {bad_argument} - goto exit; - }}}} }}}} """, argname=argname, @@ -4650,10 +4640,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st {bad_argument} goto exit; }}}} - if (!PyBuffer_IsContiguous(&{paramname}, 'C')) {{{{ - {bad_argument2} - goto exit; - }}}} """, argname=argname, bad_argument=self.bad_argument(displayname, 'read-write bytes-like object', limited_capi=limited_capi), diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 78b0c08d25ab01..5ef55524c11be2 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -1065,9 +1065,9 @@ def _f_nlocalsplus(self): def _f_lasti(self): codeunit_p = gdb.lookup_type("_Py_CODEUNIT").pointer() - prev_instr = self._gdbval["prev_instr"] + instr_ptr = self._gdbval["instr_ptr"] first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) - return int(prev_instr - first_instr) + return int(instr_ptr - first_instr) def is_shim(self): return self._f_special("owner", int) == FRAME_OWNED_BY_CSTACK diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index fc5fa1a696e26d..8c51293cf665c7 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,6 +1,6 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.6.0 +mypy==1.6.1 # needed for peg_generator: types-psutil==5.9.5.17 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index bdca51df3dac53..165b9b40566a2d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -2,910 +2,1172 @@ default stats folders. """ +from __future__ import annotations + # NOTE: Bytecode introspection modules (opcode, dis, etc.) should only -# happen when loading a single dataset. When comparing datasets, it +# be imported when loading a single dataset. When comparing datasets, it # could get it wrong, leading to subtle errors. import argparse import collections -import json -import os.path +from collections.abc import KeysView from datetime import date +import enum +import functools import itertools -import sys +import json +from operator import itemgetter +import os +from pathlib import Path import re +import sys +from typing import Any, Callable, TextIO, TypeAlias + + +RawData: TypeAlias = dict[str, Any] +Rows: TypeAlias = list[tuple] +Columns: TypeAlias = tuple[str, ...] +RowCalculator: TypeAlias = Callable[["Stats"], Rows] + + +# TODO: Check for parity + if os.name == "nt": DEFAULT_DIR = "c:\\temp\\py_stats\\" else: DEFAULT_DIR = "/tmp/py_stats/" + +SOURCE_DIR = Path(__file__).parents[2] + + TOTAL = "specialization.hit", "specialization.miss", "execution_count" -def format_ratio(num, den): - """ - Format a ratio as a percentage. When the denominator is 0, returns the empty - string. - """ - if den == 0: - return "" - else: - return f"{num/den:.01%}" +def pretty(name: str) -> str: + return name.replace("_", " ").lower() -def percentage_to_float(s): - """ - Converts a percentage string to a float. The empty string is returned as 0.0 - """ - if s == "": - return 0.0 - else: - assert s[-1] == "%" - return float(s[:-1]) +def _load_metadata_from_source(): + def get_defines(filepath: Path, prefix: str = "SPEC_FAIL"): + with open(SOURCE_DIR / filepath) as spec_src: + defines = collections.defaultdict(list) + start = "#define " + prefix + "_" + for line in spec_src: + line = line.strip() + if not line.startswith(start): + continue + line = line[len(start) :] + name, val = line.split() + defines[int(val.strip())].append(name.strip()) + return defines + + import opcode + + return { + "_specialized_instructions": [ + op for op in opcode._specialized_opmap.keys() if "__" not in op # type: ignore + ], + "_stats_defines": get_defines( + Path("Include") / "cpython" / "pystats.h", "EVAL_CALL" + ), + "_defines": get_defines(Path("Python") / "specialize.c"), + } + + +def load_raw_data(input: Path) -> RawData: + if input.is_file(): + with open(input, "r") as fd: + data = json.load(fd) + data["_stats_defines"] = {int(k): v for k, v in data["_stats_defines"].items()} + data["_defines"] = {int(k): v for k, v in data["_defines"].items()} -def join_rows(a_rows, b_rows): - """ - Joins two tables together, side-by-side, where the first column in each is a - common key. - """ - if len(a_rows) == 0 and len(b_rows) == 0: - return [] + return data - if len(a_rows): - a_ncols = list(set(len(x) for x in a_rows)) - if len(a_ncols) != 1: - raise ValueError("Table a is ragged") + elif input.is_dir(): + stats = collections.Counter[str]() - if len(b_rows): - b_ncols = list(set(len(x) for x in b_rows)) - if len(b_ncols) != 1: - raise ValueError("Table b is ragged") + for filename in input.iterdir(): + with open(filename) as fd: + for line in fd: + try: + key, value = line.split(":") + except ValueError: + print( + f"Unparsable line: '{line.strip()}' in {filename}", + file=sys.stderr, + ) + continue + stats[key.strip()] += int(value) + stats["__nfiles__"] += 1 - if len(a_rows) and len(b_rows) and a_ncols[0] != b_ncols[0]: - raise ValueError("Tables have different widths") + data = dict(stats) + data.update(_load_metadata_from_source()) + return data - if len(a_rows): - ncols = a_ncols[0] else: - ncols = b_ncols[0] + raise ValueError(f"{input:r} is not a file or directory path") - default = [""] * (ncols - 1) - a_data = {x[0]: x[1:] for x in a_rows} - b_data = {x[0]: x[1:] for x in b_rows} - if len(a_data) != len(a_rows) or len(b_data) != len(b_rows): - raise ValueError("Duplicate keys") +def save_raw_data(data: RawData, json_output: TextIO): + json.dump(data, json_output) - # To preserve ordering, use A's keys as is and then add any in B that aren't - # in A - keys = list(a_data.keys()) + [k for k in b_data.keys() if k not in a_data] - return [(k, *a_data.get(k, default), *b_data.get(k, default)) for k in keys] +class OpcodeStats: + """ + Manages the data related to specific set of opcodes, e.g. tier1 (with prefix + "opcode") or tier2 (with prefix "uops"). + """ -def calculate_specialization_stats(family_stats, total): - rows = [] - for key in sorted(family_stats): - if key.startswith("specialization.failure_kinds"): - continue - if key in ("specialization.hit", "specialization.miss"): - label = key[len("specialization.") :] - elif key == "execution_count": - continue - elif key in ( - "specialization.success", - "specialization.failure", - "specializable", - ): - continue - elif key.startswith("pair"): - continue - else: - label = key - rows.append( - ( - f"{label:>12}", - f"{family_stats[key]:>12}", - format_ratio(family_stats[key], total), - ) + def __init__(self, data: dict[str, Any], defines, specialized_instructions): + self._data = data + self._defines = defines + self._specialized_instructions = specialized_instructions + + def get_opcode_names(self) -> KeysView[str]: + return self._data.keys() + + def get_pair_counts(self) -> dict[tuple[str, str], int]: + pair_counts = {} + for name_i, opcode_stat in self._data.items(): + for key, value in opcode_stat.items(): + if value and key.startswith("pair_count"): + name_j, _, _ = key[len("pair_count") + 1 :].partition("]") + pair_counts[(name_i, name_j)] = value + return pair_counts + + def get_total_execution_count(self) -> int: + return sum(x.get("execution_count", 0) for x in self._data.values()) + + def get_execution_counts(self) -> dict[str, tuple[int, int]]: + counts = {} + for name, opcode_stat in self._data.items(): + if "execution_count" in opcode_stat: + count = opcode_stat["execution_count"] + miss = 0 + if "specializable" not in opcode_stat: + miss = opcode_stat.get("specialization.miss", 0) + counts[name] = (count, miss) + return counts + + @functools.cache + def _get_pred_succ( + self, + ) -> tuple[dict[str, collections.Counter], dict[str, collections.Counter]]: + pair_counts = self.get_pair_counts() + + predecessors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter + ) + successors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter ) - return rows + for (first, second), count in pair_counts.items(): + if count: + predecessors[second][first] = count + successors[first][second] = count + return predecessors, successors -def calculate_specialization_success_failure(family_stats): - total_attempts = 0 - for key in ("specialization.success", "specialization.failure"): - total_attempts += family_stats.get(key, 0) - rows = [] - if total_attempts: + def get_predecessors(self, opcode: str) -> collections.Counter[str]: + return self._get_pred_succ()[0][opcode] + + def get_successors(self, opcode: str) -> collections.Counter[str]: + return self._get_pred_succ()[1][opcode] + + def _get_stats_for_opcode(self, opcode: str) -> dict[str, int]: + return self._data[opcode] + + def get_specialization_total(self, opcode: str) -> int: + family_stats = self._get_stats_for_opcode(opcode) + return sum(family_stats.get(kind, 0) for kind in TOTAL) + + def get_specialization_counts(self, opcode: str) -> dict[str, int]: + family_stats = self._get_stats_for_opcode(opcode) + + result = {} + for key, value in sorted(family_stats.items()): + if key.startswith("specialization."): + label = key[len("specialization.") :] + if label in ("success", "failure") or label.startswith("failure_kinds"): + continue + elif key in ( + "execution_count", + "specializable", + ) or key.startswith("pair"): + continue + else: + label = key + result[label] = value + + return result + + def get_specialization_success_failure(self, opcode: str) -> dict[str, int]: + family_stats = self._get_stats_for_opcode(opcode) + result = {} for key in ("specialization.success", "specialization.failure"): label = key[len("specialization.") :] - label = label[0].upper() + label[1:] val = family_stats.get(key, 0) - rows.append((label, val, format_ratio(val, total_attempts))) - return rows - - -def calculate_specialization_failure_kinds(name, family_stats, defines): - total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [0] * 40 - for key in family_stats: - if not key.startswith("specialization.failure_kind"): - continue - _, index = key[:-1].split("[") - index = int(index) - failure_kinds[index] = family_stats[key] - failures = [(value, index) for (index, value) in enumerate(failure_kinds)] - failures.sort(reverse=True) - rows = [] - for value, index in failures: - if not value: - continue - rows.append( - ( - kind_to_text(index, defines, name), - value, - format_ratio(value, total_failures), - ) + result[label] = val + return result + + def get_specialization_failure_total(self, opcode: str) -> int: + return self._get_stats_for_opcode(opcode).get("specialization.failure", 0) + + def get_specialization_failure_kinds(self, opcode: str) -> dict[str, int]: + def kind_to_text(kind: int, opcode: str): + if kind <= 8: + return pretty(self._defines[kind][0]) + if opcode == "LOAD_SUPER_ATTR": + opcode = "SUPER" + elif opcode.endswith("ATTR"): + opcode = "ATTR" + elif opcode in ("FOR_ITER", "SEND"): + opcode = "ITER" + elif opcode.endswith("SUBSCR"): + opcode = "SUBSCR" + for name in self._defines[kind]: + if name.startswith(opcode): + return pretty(name[len(opcode) + 1 :]) + return "kind " + str(kind) + + family_stats = self._get_stats_for_opcode(opcode) + failure_kinds = [0] * 40 + for key in family_stats: + if not key.startswith("specialization.failure_kind"): + continue + index = int(key[:-1].split("[")[1]) + failure_kinds[index] = family_stats[key] + return { + kind_to_text(index, opcode): value + for (index, value) in enumerate(failure_kinds) + if value + } + + def is_specializable(self, opcode: str) -> bool: + return "specializable" in self._get_stats_for_opcode(opcode) + + def get_specialized_total_counts(self) -> tuple[int, int, int]: + basic = 0 + specialized_hits = 0 + specialized_misses = 0 + not_specialized = 0 + for opcode, opcode_stat in self._data.items(): + if "execution_count" not in opcode_stat: + continue + count = opcode_stat["execution_count"] + if "specializable" in opcode_stat: + not_specialized += count + elif opcode in self._specialized_instructions: + miss = opcode_stat.get("specialization.miss", 0) + specialized_hits += count - miss + specialized_misses += miss + else: + basic += count + return basic, specialized_hits, specialized_misses, not_specialized + + def get_deferred_counts(self) -> dict[str, int]: + return { + opcode: opcode_stat.get("specialization.deferred", 0) + for opcode, opcode_stat in self._data.items() + if opcode != "RESUME" + } + + def get_misses_counts(self) -> dict[str, int]: + return { + opcode: opcode_stat.get("specialization.miss", 0) + for opcode, opcode_stat in self._data.items() + if not self.is_specializable(opcode) + } + + def get_opcode_counts(self) -> dict[str, int]: + counts = {} + for opcode, entry in self._data.items(): + count = entry.get("count", 0) + if count: + counts[opcode] = count + return counts + + +class Stats: + def __init__(self, data: RawData): + self._data = data + + def get(self, key: str) -> int: + return self._data.get(key, 0) + + @functools.cache + def get_opcode_stats(self, prefix: str) -> OpcodeStats: + opcode_stats = collections.defaultdict[str, dict](dict) + for key, value in self._data.items(): + if not key.startswith(prefix): + continue + name, _, rest = key[len(prefix) + 1 :].partition("]") + opcode_stats[name][rest.strip(".")] = value + return OpcodeStats( + opcode_stats, + self._data["_defines"], + self._data["_specialized_instructions"], ) - return rows - - -def print_specialization_stats(name, family_stats, defines): - if "specializable" not in family_stats: - return - total = sum(family_stats.get(kind, 0) for kind in TOTAL) - if total == 0: - return - with Section(name, 3, f"specialization stats for {name} family"): - rows = calculate_specialization_stats(family_stats, total) - emit_table(("Kind", "Count", "Ratio"), rows) - rows = calculate_specialization_success_failure(family_stats) - if rows: - print_title("Specialization attempts", 4) - emit_table(("", "Count:", "Ratio:"), rows) - rows = calculate_specialization_failure_kinds(name, family_stats, defines) - emit_table(("Failure kind", "Count:", "Ratio:"), rows) - - -def print_comparative_specialization_stats( - name, base_family_stats, head_family_stats, defines -): - if "specializable" not in base_family_stats: - return - - base_total = sum(base_family_stats.get(kind, 0) for kind in TOTAL) - head_total = sum(head_family_stats.get(kind, 0) for kind in TOTAL) - if base_total + head_total == 0: - return - with Section(name, 3, f"specialization stats for {name} family"): - base_rows = calculate_specialization_stats(base_family_stats, base_total) - head_rows = calculate_specialization_stats(head_family_stats, head_total) - emit_table( - ("Kind", "Base Count", "Base Ratio", "Head Count", "Head Ratio"), - join_rows(base_rows, head_rows), + + def get_call_stats(self) -> dict[str, int]: + defines = self._data["_stats_defines"] + result = {} + for key, value in sorted(self._data.items()): + if "Calls to" in key: + result[key] = value + elif key.startswith("Calls "): + name, index = key[:-1].split("[") + label = f"{name} ({pretty(defines[int(index)][0])})" + result[label] = value + + for key, value in sorted(self._data.items()): + if key.startswith("Frame"): + result[key] = value + + return result + + def get_object_stats(self) -> dict[str, tuple[int, int]]: + total_materializations = self._data.get("Object new values", 0) + total_allocations = self._data.get("Object allocations", 0) + self._data.get( + "Object allocations from freelist", 0 ) - base_rows = calculate_specialization_success_failure(base_family_stats) - head_rows = calculate_specialization_success_failure(head_family_stats) - rows = join_rows(base_rows, head_rows) - if rows: - print_title("Specialization attempts", 4) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows - ) - base_rows = calculate_specialization_failure_kinds( - name, base_family_stats, defines - ) - head_rows = calculate_specialization_failure_kinds( - name, head_family_stats, defines - ) - emit_table( - ( - "Failure kind", - "Base Count:", - "Base Ratio:", - "Head Count:", - "Head Ratio:", - ), - join_rows(base_rows, head_rows), - ) + total_increfs = self._data.get( + "Object interpreter increfs", 0 + ) + self._data.get("Object increfs", 0) + total_decrefs = self._data.get( + "Object interpreter decrefs", 0 + ) + self._data.get("Object decrefs", 0) + + result = {} + for key, value in self._data.items(): + if key.startswith("Object"): + if "materialize" in key: + den = total_materializations + elif "allocations" in key: + den = total_allocations + elif "increfs" in key: + den = total_increfs + elif "decrefs" in key: + den = total_decrefs + else: + den = None + label = key[6:].strip() + label = label[0].upper() + label[1:] + result[label] = (value, den) + return result + + def get_gc_stats(self) -> list[dict[str, int]]: + gc_stats: list[dict[str, int]] = [] + for key, value in self._data.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return gc_stats + + def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: + if "Optimization attempts" not in self._data: + return {} + + attempts = self._data["Optimization attempts"] + created = self._data["Optimization traces created"] + executed = self._data["Optimization traces executed"] + uops = self._data["Optimization uops executed"] + trace_stack_overflow = self._data["Optimization trace stack overflow"] + trace_stack_underflow = self._data["Optimization trace stack underflow"] + trace_too_long = self._data["Optimization trace too long"] + trace_too_short = self._data["Optimization trace too short"] + inner_loop = self._data["Optimization inner loop"] + recursive_call = self._data["Optimization recursive call"] + + return { + "Optimization attempts": (attempts, None), + "Traces created": (created, attempts), + "Trace stack overflow": (trace_stack_overflow, attempts), + "Trace stack underflow": (trace_stack_underflow, attempts), + "Trace too long": (trace_too_long, attempts), + "Trace too short": (trace_too_short, attempts), + "Inner loop found": (inner_loop, attempts), + "Recursive call": (recursive_call, attempts), + "Traces executed": (executed, None), + "Uops executed": (uops, executed), + } + def get_histogram(self, prefix: str) -> list[tuple[int, int]]: + rows = [] + for k, v in self._data.items(): + match = re.match(f"{prefix}\\[([0-9]+)\\]", k) + if match is not None: + entry = int(match.groups()[0]) + rows.append((entry, v)) + rows.sort() + return rows + + +class Count(int): + def markdown(self) -> str: + return format(self, ",d") + + +class Ratio: + def __init__(self, num: int, den: int | None, percentage: bool = True): + self.num = num + self.den = den + self.percentage = percentage + if den == 0 and num != 0: + raise ValueError("Invalid denominator") + + def __float__(self): + if self.den == 0: + return 0.0 + elif self.den is None: + return self.num + else: + return self.num / self.den -def gather_stats(input): - # Note the output of this function must be JSON-serializable + def markdown(self) -> str: + if self.den == 0 or self.den is None: + return "" + elif self.percentage: + return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" - if os.path.isfile(input): - with open(input, "r") as fd: - stats = json.load(fd) - stats["_stats_defines"] = { - int(k): v for k, v in stats["_stats_defines"].items() - } - stats["_defines"] = {int(k): v for k, v in stats["_defines"].items()} - return stats +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) - elif os.path.isdir(input): - stats = collections.Counter() - for filename in os.listdir(input): - with open(os.path.join(input, filename)) as fd: - for line in fd: - try: - key, value = line.split(":") - except ValueError: - print( - f"Unparsable line: '{line.strip()}' in {filename}", - file=sys.stderr, - ) - continue - key = key.strip() - value = int(value) - stats[key] += value - stats["__nfiles__"] += 1 - import opcode +class JoinMode(enum.Enum): + # Join using the first column as a key + SIMPLE = 0 + # Join using the first column as a key, and indicate the change in the + # second column of each input table as a new column + CHANGE = 1 + # Join using the first column as a key, indicating the change in the second + # column of each input table as a ne column, and omit all other columns + CHANGE_ONE_COLUMN = 2 - stats["_specialized_instructions"] = [ - op for op in opcode._specialized_opmap.keys() if "__" not in op - ] - stats["_stats_defines"] = get_stats_defines() - stats["_defines"] = get_defines() - return stats - else: - raise ValueError(f"{input:r} is not a file or directory path") +class Table: + """ + A Table defines how to convert a set of Stats into a specific set of rows + displaying some aspect of the data. + """ + def __init__( + self, + column_names: Columns, + calc_rows: RowCalculator, + join_mode: JoinMode = JoinMode.SIMPLE, + ): + self.columns = column_names + self.calc_rows = calc_rows + self.join_mode = join_mode + + def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: + match self.join_mode: + case JoinMode.SIMPLE: + return (key, *row_a, *row_b) + case JoinMode.CHANGE: + return (key, *row_a, *row_b, DiffRatio(row_a[0], row_b[0])) + case JoinMode.CHANGE_ONE_COLUMN: + return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0])) + + def join_columns(self, columns: Columns) -> Columns: + match self.join_mode: + case JoinMode.SIMPLE: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), + ) + case JoinMode.CHANGE: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), + ) + ("Change:",) + case JoinMode.CHANGE_ONE_COLUMN: + return ( + columns[0], + "Base " + columns[1], + "Head " + columns[1], + "Change:", + ) -def extract_opcode_stats(stats, prefix): - opcode_stats = collections.defaultdict(dict) - for key, value in stats.items(): - if not key.startswith(prefix): - continue - name, _, rest = key[len(prefix) + 1 :].partition("]") - opcode_stats[name][rest.strip(".")] = value - return opcode_stats - - -def parse_kinds(spec_src, prefix="SPEC_FAIL"): - defines = collections.defaultdict(list) - start = "#define " + prefix + "_" - for line in spec_src: - line = line.strip() - if not line.startswith(start): - continue - line = line[len(start) :] - name, val = line.split() - defines[int(val.strip())].append(name.strip()) - return defines - - -def pretty(defname): - return defname.replace("_", " ").lower() - - -def kind_to_text(kind, defines, opname): - if kind <= 8: - return pretty(defines[kind][0]) - if opname == "LOAD_SUPER_ATTR": - opname = "SUPER" - elif opname.endswith("ATTR"): - opname = "ATTR" - elif opname in ("FOR_ITER", "SEND"): - opname = "ITER" - elif opname.endswith("SUBSCR"): - opname = "SUBSCR" - for name in defines[kind]: - if name.startswith(opname): - return pretty(name[len(opname) + 1 :]) - return "kind " + str(kind) - - -def categorized_counts(opcode_stats, specialized_instructions): - basic = 0 - specialized = 0 - not_specialized = 0 - for name, opcode_stat in opcode_stats.items(): - if "execution_count" not in opcode_stat: - continue - count = opcode_stat["execution_count"] - if "specializable" in opcode_stat: - not_specialized += count - elif name in specialized_instructions: - miss = opcode_stat.get("specialization.miss", 0) - not_specialized += miss - specialized += count - miss - else: - basic += count - return basic, not_specialized, specialized + def join_tables(self, rows_a: Rows, rows_b: Rows) -> tuple[Columns, Rows]: + ncols = len(self.columns) + default = ("",) * (ncols - 1) + data_a = {x[0]: x[1:] for x in rows_a} + data_b = {x[0]: x[1:] for x in rows_b} -def print_title(name, level=2): - print("#" * level, name) - print() + if len(data_a) != len(rows_a) or len(data_b) != len(rows_b): + raise ValueError("Duplicate keys") + + # To preserve ordering, use A's keys as is and then add any in B that + # aren't in A + keys = list(data_a.keys()) + [k for k in data_b.keys() if k not in data_a] + rows = [ + self.join_row(k, data_a.get(k, default), data_b.get(k, default)) + for k in keys + ] + if self.join_mode in (JoinMode.CHANGE, JoinMode.CHANGE_ONE_COLUMN): + rows.sort(key=lambda row: abs(float(row[-1])), reverse=True) + + columns = self.join_columns(self.columns) + return columns, rows + + def get_table( + self, base_stats: Stats, head_stats: Stats | None = None + ) -> tuple[Columns, Rows]: + if head_stats is None: + rows = self.calc_rows(base_stats) + return self.columns, rows + else: + rows_a = self.calc_rows(base_stats) + rows_b = self.calc_rows(head_stats) + cols, rows = self.join_tables(rows_a, rows_b) + return cols, rows class Section: - def __init__(self, title, level=2, summary=None): + """ + A Section defines a section of the output document. + """ + + def __init__( + self, + title: str = "", + summary: str = "", + part_iter=None, + comparative: bool = True, + ): self.title = title - self.level = level - if summary is None: + if not summary: self.summary = title.lower() else: self.summary = summary + if part_iter is None: + part_iter = [] + if isinstance(part_iter, list): - def __enter__(self): - print_title(self.title, self.level) - print("
") - print("", self.summary, "") - print() - return self + def iter_parts(base_stats: Stats, head_stats: Stats | None): + yield from part_iter - def __exit__(*args): - print() - print("
") - print() + self.part_iter = iter_parts + else: + self.part_iter = part_iter + self.comparative = comparative -def to_str(x): - if isinstance(x, int): - return format(x, ",d") - else: - return str(x) - - -def emit_table(header, rows): - width = len(header) - header_line = "|" - under_line = "|" - for item in header: - under = "---" - if item.endswith(":"): - item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line) - print(under_line) - for row in rows: - if width is not None and len(row) != width: - raise ValueError("Wrong number of elements in row '" + str(row) + "'") - print("|", " | ".join(to_str(i) for i in row), "|") - print() - - -def emit_histogram(title, stats, key, total): - rows = [] - for k, v in stats.items(): - if k.startswith(key): - entry = int(re.match(r".+\[([0-9]+)\]", k).groups()[0]) - rows.append((f"<= {entry}", int(v), format_ratio(int(v), total))) - # Don't include larger buckets with 0 entries - for j in range(len(rows) - 1, -1, -1): - if rows[j][1] != 0: - break - rows = rows[: j + 1] - - print(f"**{title}**\n") - emit_table(("Range", "Count:", "Ratio:"), rows) - - -def calculate_execution_counts(opcode_stats, total): - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - miss = 0 - if "specializable" not in opcode_stat: - miss = opcode_stat.get("specialization.miss") - counts.append((count, name, miss)) - counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, name, miss in counts: - cumulative += count - if miss: - miss = format_ratio(miss, count) - else: - miss = "" - rows.append( - ( - name, - count, - format_ratio(count, total), - format_ratio(cumulative, total), - miss, +def calc_execution_count_table(prefix: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats(prefix) + counts = opcode_stats.get_execution_counts() + total = opcode_stats.get_total_execution_count() + cumulative = 0 + rows: Rows = [] + for opcode, (count, miss) in sorted( + counts.items(), key=itemgetter(1), reverse=True + ): + cumulative += count + if miss: + miss_val = Ratio(miss, count) + else: + miss_val = None + rows.append( + ( + opcode, + Count(count), + Ratio(count, total), + Ratio(cumulative, total), + miss_val, + ) ) - ) - return rows + return rows + return calc -def emit_execution_counts(opcode_stats, total): - with Section("Execution counts", summary="execution counts for all instructions"): - rows = calculate_execution_counts(opcode_stats, total) - emit_table(("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), rows) +def execution_count_section() -> Section: + return Section( + "Execution counts", + "execution counts for all instructions", + [ + Table( + ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), + calc_execution_count_table("opcode"), + join_mode=JoinMode.CHANGE_ONE_COLUMN, + ) + ], + ) -def _emit_comparative_execution_counts(base_rows, head_rows): - base_data = {x[0]: x[1:] for x in base_rows} - head_data = {x[0]: x[1:] for x in head_rows} - opcodes = base_data.keys() | head_data.keys() - rows = [] - default = [0, "0.0%", "0.0%", 0] - for opcode in opcodes: - base_entry = base_data.get(opcode, default) - head_entry = head_data.get(opcode, default) - if base_entry[0] == 0: - change = 1 - else: - change = (head_entry[0] - base_entry[0]) / base_entry[0] - rows.append((opcode, base_entry[0], head_entry[0], f"{change:0.1%}")) +def pair_count_section() -> Section: + def calc_pair_count_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + pair_counts = opcode_stats.get_pair_counts() + total = opcode_stats.get_total_execution_count() - rows.sort(key=lambda x: abs(percentage_to_float(x[-1])), reverse=True) + cumulative = 0 + rows: Rows = [] + for (opcode_i, opcode_j), count in itertools.islice( + sorted(pair_counts.items(), key=itemgetter(1), reverse=True), 100 + ): + cumulative += count + rows.append( + ( + f"{opcode_i} {opcode_j}", + Count(count), + Ratio(count, total), + Ratio(cumulative, total), + ) + ) + return rows + + return Section( + "Pair counts", + "Pair counts for top 100 pairs", + [ + Table( + ("Pair", "Count:", "Self:", "Cumulative:"), + calc_pair_count_table, + ) + ], + comparative=False, + ) - emit_table(("Name", "Base Count:", "Head Count:", "Change:"), rows) +def pre_succ_pairs_section() -> Section: + def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = None): + assert head_stats is None -def emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total, level=2 -): - with Section( - "Execution counts", summary="execution counts for all instructions", level=level - ): - base_rows = calculate_execution_counts(base_opcode_stats, base_total) - head_rows = calculate_execution_counts(head_opcode_stats, head_total) - _emit_comparative_execution_counts(base_rows, head_rows) + opcode_stats = base_stats.get_opcode_stats("opcode") + for opcode in opcode_stats.get_opcode_names(): + predecessors = opcode_stats.get_predecessors(opcode) + successors = opcode_stats.get_successors(opcode) + predecessors_total = predecessors.total() + successors_total = successors.total() + if predecessors_total == 0 and successors_total == 0: + continue + pred_rows = [ + (pred, Count(count), Ratio(count, predecessors_total)) + for (pred, count) in predecessors.most_common(5) + ] + succ_rows = [ + (succ, Count(count), Ratio(count, successors_total)) + for (succ, count) in successors.most_common(5) + ] + + yield Section( + opcode, + f"Successors and predecessors for {opcode}", + [ + Table( + ("Predecessors", "Count:", "Percentage:"), + lambda *_: pred_rows, # type: ignore + ), + Table( + ("Successors", "Count:", "Percentage:"), + lambda *_: succ_rows, # type: ignore + ), + ], + ) -def get_defines(): - spec_path = os.path.join(os.path.dirname(__file__), "../../Python/specialize.c") - with open(spec_path) as spec_src: - defines = parse_kinds(spec_src) - return defines + return Section( + "Predecessor/Successor Pairs", + "Top 5 predecessors and successors of each opcode", + iter_pre_succ_pairs_tables, + comparative=False, + ) -def emit_specialization_stats(opcode_stats, defines): - with Section("Specialization stats", summary="specialization stats by family"): - for name, opcode_stat in opcode_stats.items(): - print_specialization_stats(name, opcode_stat, defines) +def specialization_section() -> Section: + def calc_specialization_table(opcode: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + total = opcode_stats.get_specialization_total(opcode) + specialization_counts = opcode_stats.get_specialization_counts(opcode) + + return [ + ( + f"{label:>12}", + Count(count), + Ratio(count, total), + ) + for label, count in specialization_counts.items() + ] + return calc -def emit_comparative_specialization_stats( - base_opcode_stats, head_opcode_stats, defines -): - with Section("Specialization stats", summary="specialization stats by family"): - opcodes = set(base_opcode_stats.keys()) & set(head_opcode_stats.keys()) - for opcode in opcodes: - print_comparative_specialization_stats( - opcode, base_opcode_stats[opcode], head_opcode_stats[opcode], defines + def calc_specialization_success_failure_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + values = stats.get_opcode_stats( + "opcode" + ).get_specialization_success_failure(name) + total = sum(values.values()) + if total: + return [ + (label.capitalize(), Count(val), Ratio(val, total)) + for label, val in values.items() + ] + else: + return [] + + return calc + + def calc_specialization_failure_kind_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + failures = opcode_stats.get_specialization_failure_kinds(name) + total = opcode_stats.get_specialization_failure_total(name) + + return sorted( + [ + (label, Count(value), Ratio(value, total)) + for label, value in failures.items() + if value + ], + key=itemgetter(1), + reverse=True, ) + return calc + + def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None): + opcode_base_stats = base_stats.get_opcode_stats("opcode") + names = opcode_base_stats.get_opcode_names() + if head_stats is not None: + opcode_head_stats = head_stats.get_opcode_stats("opcode") + names &= opcode_head_stats.get_opcode_names() # type: ignore + else: + opcode_head_stats = None -def calculate_specialization_effectiveness( - opcode_stats, total, specialized_instructions -): - basic, not_specialized, specialized = categorized_counts( - opcode_stats, specialized_instructions + for opcode in sorted(names): + if not opcode_base_stats.is_specializable(opcode): + continue + if opcode_base_stats.get_specialization_total(opcode) == 0 and ( + opcode_head_stats is None + or opcode_head_stats.get_specialization_total(opcode) == 0 + ): + continue + yield Section( + opcode, + f"specialization stats for {opcode} family", + [ + Table( + ("Kind", "Count:", "Ratio:"), + calc_specialization_table(opcode), + JoinMode.CHANGE, + ), + Table( + ("", "Count:", "Ratio:"), + calc_specialization_success_failure_table(opcode), + JoinMode.CHANGE, + ), + Table( + ("Failure kind", "Count:", "Ratio:"), + calc_specialization_failure_kind_table(opcode), + JoinMode.CHANGE, + ), + ], + ) + + return Section( + "Specialization stats", + "specialization stats by family", + iter_specialization_tables, ) - return [ - ("Basic", basic, format_ratio(basic, total)), - ("Not specialized", not_specialized, format_ratio(not_specialized, total)), - ("Specialized", specialized, format_ratio(specialized, total)), - ] -def emit_specialization_overview(opcode_stats, total, specialized_instructions): - with Section("Specialization effectiveness"): - rows = calculate_specialization_effectiveness( - opcode_stats, total, specialized_instructions - ) - emit_table(("Instructions", "Count:", "Ratio:"), rows) - for title, field in ( - ("Deferred", "specialization.deferred"), - ("Misses", "specialization.miss"), - ): - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - # Avoid double counting misses - if title == "Misses" and "specializable" in opcode_stat: - continue - value = opcode_stat.get(field, 0) - counts.append((value, name)) - total += value - counts.sort(reverse=True) - if total: - with Section(f"{title} by instruction", 3): - rows = [ - (name, count, format_ratio(count, total)) - for (count, name) in counts[:10] - ] - emit_table(("Name", "Count:", "Ratio:"), rows) - - -def emit_comparative_specialization_overview( - base_opcode_stats, - base_total, - head_opcode_stats, - head_total, - specialized_instructions, -): - with Section("Specialization effectiveness"): - base_rows = calculate_specialization_effectiveness( - base_opcode_stats, base_total, specialized_instructions - ) - head_rows = calculate_specialization_effectiveness( - head_opcode_stats, head_total, specialized_instructions - ) - emit_table( +def specialization_effectiveness_section() -> Section: + def calc_specialization_effectiveness_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + total = opcode_stats.get_total_execution_count() + + ( + basic, + specialized_hits, + specialized_misses, + not_specialized, + ) = opcode_stats.get_specialized_total_counts() + + return [ + ("Basic", Count(basic), Ratio(basic, total)), ( - "Instructions", - "Base Count:", - "Base Ratio:", - "Head Count:", - "Head Ratio:", + "Not specialized", + Count(not_specialized), + Ratio(not_specialized, total), ), - join_rows(base_rows, head_rows), - ) + ( + "Specialized hits", + Count(specialized_hits), + Ratio(specialized_hits, total), + ), + ( + "Specialized misses", + Count(specialized_misses), + Ratio(specialized_misses, total), + ), + ] + + def calc_deferred_by_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + deferred_counts = opcode_stats.get_deferred_counts() + total = sum(deferred_counts.values()) + if total == 0: + return [] + + return [ + (name, Count(value), Ratio(value, total)) + for name, value in sorted( + deferred_counts.items(), key=itemgetter(1), reverse=True + )[:10] + ] + def calc_misses_by_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + misses_counts = opcode_stats.get_misses_counts() + total = sum(misses_counts.values()) + if total == 0: + return [] + + return [ + (name, Count(value), Ratio(value, total)) + for name, value in sorted( + misses_counts.items(), key=itemgetter(1), reverse=True + )[:10] + ] -def get_stats_defines(): - stats_path = os.path.join( - os.path.dirname(__file__), "../../Include/cpython/pystats.h" + return Section( + "Specialization effectiveness", + "", + [ + Table( + ("Instructions", "Count:", "Ratio:"), + calc_specialization_effectiveness_table, + JoinMode.CHANGE, + ), + Section( + "Deferred by instruction", + "", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_deferred_by_table, + JoinMode.CHANGE, + ) + ], + ), + Section( + "Misses by instruction", + "", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_misses_by_table, + JoinMode.CHANGE, + ) + ], + ), + ], ) - with open(stats_path) as stats_src: - defines = parse_kinds(stats_src, prefix="EVAL_CALL") - return defines - - -def calculate_call_stats(stats, defines): - total = 0 - for key, value in stats.items(): - if "Calls to" in key: - total += value - rows = [] - for key, value in stats.items(): - if "Calls to" in key: - rows.append((key, value, format_ratio(value, total))) - elif key.startswith("Calls "): - name, index = key[:-1].split("[") - index = int(index) - label = name + " (" + pretty(defines[index][0]) + ")" - rows.append((label, value, format_ratio(value, total))) - for key, value in stats.items(): - if key.startswith("Frame"): - rows.append((key, value, format_ratio(value, total))) - return rows - - -def emit_call_stats(stats, defines): - with Section("Call stats", summary="Inlined calls and frame stats"): - rows = calculate_call_stats(stats, defines) - emit_table(("", "Count:", "Ratio:"), rows) - - -def emit_comparative_call_stats(base_stats, head_stats, defines): - with Section("Call stats", summary="Inlined calls and frame stats"): - base_rows = calculate_call_stats(base_stats, defines) - head_rows = calculate_call_stats(head_stats, defines) - rows = join_rows(base_rows, head_rows) - rows.sort(key=lambda x: -percentage_to_float(x[-1])) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows - ) -def calculate_object_stats(stats): - total_materializations = stats.get("Object new values") - total_allocations = stats.get("Object allocations") + stats.get( - "Object allocations from freelist" - ) - total_increfs = stats.get("Object interpreter increfs") + stats.get( - "Object increfs" - ) - total_decrefs = stats.get("Object interpreter decrefs") + stats.get( - "Object decrefs" +def call_stats_section() -> Section: + def calc_call_stats_table(stats: Stats) -> Rows: + call_stats = stats.get_call_stats() + total = sum(v for k, v in call_stats.items() if "Calls to" in k) + return [ + (key, Count(value), Ratio(value, total)) + for key, value in call_stats.items() + ] + + return Section( + "Call stats", + "Inlined calls and frame stats", + [ + Table( + ("", "Count:", "Ratio:"), + calc_call_stats_table, + JoinMode.CHANGE, + ) + ], ) - rows = [] - for key, value in stats.items(): - if key.startswith("Object"): - if "materialize" in key: - ratio = format_ratio(value, total_materializations) - elif "allocations" in key: - ratio = format_ratio(value, total_allocations) - elif "increfs" in key: - ratio = format_ratio(value, total_increfs) - elif "decrefs" in key: - ratio = format_ratio(value, total_decrefs) - else: - ratio = "" - label = key[6:].strip() - label = label[0].upper() + label[1:] - rows.append((label, value, ratio)) - return rows - - -def calculate_gc_stats(stats): - gc_stats = [] - for key, value in stats.items(): - if not key.startswith("GC"): - continue - n, _, rest = key[3:].partition("]") - name = rest.strip() - gen_n = int(n) - while len(gc_stats) <= gen_n: - gc_stats.append({}) - gc_stats[gen_n][name] = value - return [ - (i, gen["collections"], gen["objects collected"], gen["object visits"]) - for (i, gen) in enumerate(gc_stats) - ] - - -def emit_object_stats(stats): - with Section("Object stats", summary="allocations, frees and dict materializatons"): - rows = calculate_object_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) - - -def emit_comparative_object_stats(base_stats, head_stats): - with Section("Object stats", summary="allocations, frees and dict materializatons"): - base_rows = calculate_object_stats(base_stats) - head_rows = calculate_object_stats(head_stats) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), - join_rows(base_rows, head_rows), - ) -def emit_gc_stats(stats): - with Section("GC stats", summary="GC collections and effectiveness"): - rows = calculate_gc_stats(stats) - emit_table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), - rows, - ) +def object_stats_section() -> Section: + def calc_object_stats_table(stats: Stats) -> Rows: + object_stats = stats.get_object_stats() + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in object_stats.items() + ] + return Section( + "Object stats", + "allocations, frees and dict materializatons", + [ + Table( + ("", "Count:", "Ratio:"), + calc_object_stats_table, + JoinMode.CHANGE, + ) + ], + ) -def emit_comparative_gc_stats(base_stats, head_stats): - with Section("GC stats", summary="GC collections and effectiveness"): - base_rows = calculate_gc_stats(base_stats) - head_rows = calculate_gc_stats(head_stats) - emit_table( - ( - "Generation:", - "Base collections:", - "Head collections:", - "Base objects collected:", - "Head objects collected:", - "Base object visits:", - "Head object visits:", - ), - join_rows(base_rows, head_rows), - ) +def gc_stats_section() -> Section: + def calc_gc_stats(stats: Stats) -> Rows: + gc_stats = stats.get_gc_stats() -def get_total(opcode_stats): - total = 0 - for opcode_stat in opcode_stats.values(): - if "execution_count" in opcode_stat: - total += opcode_stat["execution_count"] - return total - - -def emit_pair_counts(opcode_stats, total): - pair_counts = [] - for name_i, opcode_stat in opcode_stats.items(): - for key, value in opcode_stat.items(): - if key.startswith("pair_count"): - name_j, _, _ = key[11:].partition("]") - if value: - pair_counts.append((value, (name_i, name_j))) - with Section("Pair counts", summary="Pair counts for top 100 pairs"): - pair_counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, pair in itertools.islice(pair_counts, 100): - name_i, name_j = pair - cumulative += count - rows.append( - ( - f"{name_i} {name_j}", - count, - format_ratio(count, total), - format_ratio(cumulative, total), - ) + return [ + ( + Count(i), + Count(gen["collections"]), + Count(gen["objects collected"]), + Count(gen["object visits"]), ) - emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows) - with Section( - "Predecessor/Successor Pairs", - summary="Top 5 predecessors and successors of each opcode", - ): - predecessors = collections.defaultdict(collections.Counter) - successors = collections.defaultdict(collections.Counter) - total_predecessors = collections.Counter() - total_successors = collections.Counter() - for count, (first, second) in pair_counts: - if count: - predecessors[second][first] = count - successors[first][second] = count - total_predecessors[second] += count - total_successors[first] += count - for name in opcode_stats.keys(): - total1 = total_predecessors[name] - total2 = total_successors[name] - if total1 == 0 and total2 == 0: - continue - pred_rows = succ_rows = () - if total1: - pred_rows = [ - (pred, count, f"{count/total1:.1%}") - for (pred, count) in predecessors[name].most_common(5) - ] - if total2: - succ_rows = [ - (succ, count, f"{count/total2:.1%}") - for (succ, count) in successors[name].most_common(5) - ] - with Section(name, 3, f"Successors and predecessors for {name}"): - emit_table(("Predecessors", "Count:", "Percentage:"), pred_rows) - emit_table(("Successors", "Count:", "Percentage:"), succ_rows) - - -def calculate_optimization_stats(stats): - attempts = stats["Optimization attempts"] - created = stats["Optimization traces created"] - executed = stats["Optimization traces executed"] - uops = stats["Optimization uops executed"] - trace_stack_overflow = stats["Optimization trace stack overflow"] - trace_stack_underflow = stats["Optimization trace stack underflow"] - trace_too_long = stats["Optimization trace too long"] - trace_too_short = stats["Optimiztion trace too short"] - inner_loop = stats["Optimization inner loop"] - recursive_call = stats["Optimization recursive call"] - - return [ - ("Optimization attempts", attempts, ""), - ("Traces created", created, format_ratio(created, attempts)), - ("Traces executed", executed, ""), - ("Uops executed", uops, int(uops / (executed or 1))), - ("Trace stack overflow", trace_stack_overflow, ""), - ("Trace stack underflow", trace_stack_underflow, ""), - ("Trace too long", trace_too_long, ""), - ("Trace too short", trace_too_short, ""), - ("Inner loop found", inner_loop, ""), - ("Recursive call", recursive_call, ""), - ] - - -def calculate_uop_execution_counts(opcode_stats): - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - counts.append((count, name)) - total += count - counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, name in counts: - cumulative += count - rows.append( - (name, count, format_ratio(count, total), format_ratio(cumulative, total)) - ) - return rows + for (i, gen) in enumerate(gc_stats) + ] + return Section( + "GC stats", + "GC collections and effectiveness", + [ + Table( + ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + calc_gc_stats, + ) + ], + ) -def emit_optimization_stats(stats): - if "Optimization attempts" not in stats: - return - uop_stats = extract_opcode_stats(stats, "uops") +def optimization_section() -> Section: + def calc_optimization_table(stats: Stats) -> Rows: + optimization_stats = stats.get_optimization_stats() - with Section( - "Optimization (Tier 2) stats", summary="statistics about the Tier 2 optimizer" - ): - with Section("Overall stats", level=3): - rows = calculate_optimization_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) - - emit_histogram( - "Trace length histogram", - stats, - "Trace length", - stats["Optimization traces created"], + return [ + ( + label, + Count(value), + Ratio(value, den, percentage=label != "Uops executed"), + ) + for label, (value, den) in optimization_stats.items() + ] + + def calc_histogram_table(key: str, den: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + histogram = stats.get_histogram(key) + denominator = stats.get(den) + + rows: Rows = [] + last_non_zero = 0 + for k, v in histogram: + if v != 0: + last_non_zero = len(rows) + rows.append( + ( + f"<= {k:,d}", + Count(v), + Ratio(v, denominator), + ) + ) + # Don't include any zero entries at the end + rows = rows[: last_non_zero + 1] + return rows + + return calc + + def calc_unsupported_opcodes_table(stats: Stats) -> Rows: + unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode") + return sorted( + [ + (opcode, Count(count)) + for opcode, count in unsupported_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, ) - emit_histogram( - "Optimized trace length histogram", - stats, - "Optimized trace length", - stats["Optimization traces created"], + + def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): + if not base_stats.get_optimization_stats() or ( + head_stats is not None and not head_stats.get_optimization_stats() + ): + return + + yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE) + for name, den in [ + ("Trace length", "Optimization traces created"), + ("Optimized trace length", "Optimization traces created"), + ("Trace run length", "Optimization traces executed"), + ]: + yield Section( + f"{name} histogram", + "", + [ + Table( + ("Range", "Count:", "Ratio:"), + calc_histogram_table(name, den), + JoinMode.CHANGE, + ) + ], + ) + yield Section( + "Uop stats", + "", + [ + Table( + ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), + calc_execution_count_table("uops"), + JoinMode.CHANGE_ONE_COLUMN, + ) + ], ) - emit_histogram( - "Trace run length histogram", - stats, - "Trace run length", - stats["Optimization traces executed"], + yield Section( + "Unsupported opcodes", + "", + [ + Table( + ("Opcode", "Count:"), + calc_unsupported_opcodes_table, + JoinMode.CHANGE, + ) + ], ) - with Section("Uop stats", level=3): - rows = calculate_uop_execution_counts(uop_stats) - emit_table(("Uop", "Count:", "Self:", "Cumulative:"), rows) - - with Section("Unsupported opcodes", level=3): - unsupported_opcodes = extract_opcode_stats(stats, "unsupported_opcode") - data = [] - for opcode, entry in unsupported_opcodes.items(): - data.append((entry["count"], opcode)) - data.sort(reverse=True) - rows = [(x[1], x[0]) for x in data] - emit_table(("Opcode", "Count"), rows) - + return Section( + "Optimization (Tier 2) stats", + "statistics about the Tier 2 optimizer", + iter_optimization_tables, + ) -def emit_comparative_optimization_stats(base_stats, head_stats): - print("## Comparative optimization stats not implemented\n\n") +def meta_stats_section() -> Section: + def calc_rows(stats: Stats) -> Rows: + return [("Number of data files", Count(stats.get("__nfiles__")))] -def output_single_stats(stats): - opcode_stats = extract_opcode_stats(stats, "opcode") - total = get_total(opcode_stats) - emit_execution_counts(opcode_stats, total) - emit_pair_counts(opcode_stats, total) - emit_specialization_stats(opcode_stats, stats["_defines"]) - emit_specialization_overview( - opcode_stats, total, stats["_specialized_instructions"] + return Section( + "Meta stats", + "Meta statistics", + [Table(("", "Count:"), calc_rows, JoinMode.CHANGE)], ) - emit_call_stats(stats, stats["_stats_defines"]) - emit_object_stats(stats) - emit_gc_stats(stats) - emit_optimization_stats(stats) - with Section("Meta stats", summary="Meta statistics"): - emit_table(("", "Count:"), [("Number of data files", stats["__nfiles__"])]) - -def output_comparative_stats(base_stats, head_stats): - base_opcode_stats = extract_opcode_stats(base_stats, "opcode") - base_total = get_total(base_opcode_stats) - head_opcode_stats = extract_opcode_stats(head_stats, "opcode") - head_total = get_total(head_opcode_stats) - - emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total - ) - emit_comparative_specialization_stats( - base_opcode_stats, head_opcode_stats, head_stats["_defines"] - ) - emit_comparative_specialization_overview( - base_opcode_stats, - base_total, - head_opcode_stats, - head_total, - head_stats["_specialized_instructions"], - ) - emit_comparative_call_stats(base_stats, head_stats, head_stats["_stats_defines"]) - emit_comparative_object_stats(base_stats, head_stats) - emit_comparative_gc_stats(base_stats, head_stats) - emit_comparative_optimization_stats(base_stats, head_stats) - - -def output_stats(inputs, json_output=None): - if len(inputs) == 1: - stats = gather_stats(inputs[0]) - if json_output is not None: - json.dump(stats, json_output) - output_single_stats(stats) - elif len(inputs) == 2: - if json_output is not None: - raise ValueError("Can not output to JSON when there are multiple inputs") - - base_stats = gather_stats(inputs[0]) - head_stats = gather_stats(inputs[1]) - output_comparative_stats(base_stats, head_stats) - - print("---") - print("Stats gathered on:", date.today()) +LAYOUT = [ + execution_count_section(), + pair_count_section(), + pre_succ_pairs_section(), + specialization_section(), + specialization_effectiveness_section(), + call_stats_section(), + object_stats_section(), + gc_stats_section(), + optimization_section(), + meta_stats_section(), +] + + +def output_markdown( + out: TextIO, + obj: Section | Table | list, + base_stats: Stats, + head_stats: Stats | None = None, + level: int = 2, +) -> None: + def to_markdown(x): + if hasattr(x, "markdown"): + return x.markdown() + elif isinstance(x, str): + return x + elif x is None: + return "" + else: + raise TypeError(f"Can't convert {x} to markdown") + + match obj: + case Section(): + if obj.title: + print("#" * level, obj.title, file=out) + print(file=out) + print("
", file=out) + print("", obj.summary, "", file=out) + print(file=out) + if head_stats is not None and obj.comparative is False: + print("Not included in comparative output.\n") + else: + for part in obj.part_iter(base_stats, head_stats): + output_markdown(out, part, base_stats, head_stats, level=level + 1) + print(file=out) + if obj.title: + print("
", file=out) + print(file=out) + + case Table(): + header, rows = obj.get_table(base_stats, head_stats) + if len(rows) == 0: + return + + width = len(header) + header_line = "|" + under_line = "|" + for item in header: + under = "---" + if item.endswith(":"): + item = item[:-1] + under += ":" + header_line += item + " | " + under_line += under + "|" + print(header_line, file=out) + print(under_line, file=out) + for row in rows: + if len(row) != width: + raise ValueError( + "Wrong number of elements in row '" + str(row) + "'" + ) + print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) + print(file=out) + + case list(): + for part in obj: + output_markdown(out, part, base_stats, head_stats, level=level) + + print("---", file=out) + print("Stats gathered on:", date.today(), file=out) + + +def output_stats(inputs: list[Path], json_output=TextIO | None): + match len(inputs): + case 1: + data = load_raw_data(Path(inputs[0])) + if json_output is not None: + save_raw_data(data, json_output) # type: ignore + stats = Stats(data) + output_markdown(sys.stdout, LAYOUT, stats) + case 2: + if json_output is not None: + raise ValueError( + "Can not output to JSON when there are multiple inputs" + ) + base_data = load_raw_data(Path(inputs[0])) + head_data = load_raw_data(Path(inputs[1])) + base_stats = Stats(base_data) + head_stats = Stats(head_data) + output_markdown(sys.stdout, LAYOUT, base_stats, head_stats) def main(): diff --git a/configure b/configure index c87f518382f2ec..8b90c1306f825f 100755 --- a/configure +++ b/configure @@ -9268,10 +9268,15 @@ then : # without this, configure fails to find pthread_create, sem_init, # etc because they are only available in the sysroot for # wasm32-wasi-threads. + # Note: wasi-threads requires --import-memory. + # Note: wasi requires --export-memory. + # Note: --export-memory is implicit unless --import-memory is given + # Note: this requires LLVM >= 16. as_fn_append CFLAGS " -target wasm32-wasi-threads -pthread" as_fn_append CFLAGS_NODIST " -target wasm32-wasi-threads -pthread" as_fn_append LDFLAGS_NODIST " -target wasm32-wasi-threads -pthread" as_fn_append LDFLAGS_NODIST " -Wl,--import-memory" + as_fn_append LDFLAGS_NODIST " -Wl,--export-memory" as_fn_append LDFLAGS_NODIST " -Wl,--max-memory=10485760" fi @@ -27926,6 +27931,7 @@ printf "%s\n" "$ac_cv_libatomic_needed" >&6; } if test "x$ac_cv_libatomic_needed" = xyes then : LIBS="${LIBS} -latomic" + LIBATOMIC=${LIBATOMIC-"-latomic"} fi CPPFLAGS=$save_CPPFLAGS @@ -30079,7 +30085,7 @@ fi then : - + as_fn_append MODULE_BLOCK "MODULE__TESTCAPI_LDFLAGS=$LIBATOMIC$as_nl" fi if test "$py_cv_module__testcapi" = yes; then diff --git a/configure.ac b/configure.ac index cd69f0ede54496..a4ac589822708d 100644 --- a/configure.ac +++ b/configure.ac @@ -2162,10 +2162,15 @@ AS_CASE([$ac_sys_system], # without this, configure fails to find pthread_create, sem_init, # etc because they are only available in the sysroot for # wasm32-wasi-threads. + # Note: wasi-threads requires --import-memory. + # Note: wasi requires --export-memory. + # Note: --export-memory is implicit unless --import-memory is given + # Note: this requires LLVM >= 16. AS_VAR_APPEND([CFLAGS], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([CFLAGS_NODIST], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--import-memory"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--export-memory"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--max-memory=10485760"]) ]) @@ -7051,7 +7056,8 @@ int main() ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], - [LIBS="${LIBS} -latomic"]) + [LIBS="${LIBS} -latomic" + LIBATOMIC=${LIBATOMIC-"-latomic"}]) _RESTORE_VAR([CPPFLAGS]) @@ -7323,7 +7329,10 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules -PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testcapi], + [test "$TEST_MODULES" = yes], + dnl Modules/_testcapi needs -latomic for 32bit AIX build + [], [], [$LIBATOMIC]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes])