diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index ada5fb0fe64dc2..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:40 - -ENV CC=clang - -ENV WASI_SDK_VERSION=24 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=22.0.0 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..64c85c1101e6e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 680f2ed5be031a..7e9c3caf23f079 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -207,7 +207,6 @@ Doc/c-api/stable.rst @encukou **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith diff --git a/Doc/Makefile b/Doc/Makefile index a2d89343648dc1..a090ee5ba92705 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -305,13 +305,15 @@ serve: # for development releases: always build .PHONY: autobuild-dev +autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev: - $(MAKE) dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) -# for quick rebuilds (HTML only) +# for HTML-only rebuilds .PHONY: autobuild-dev-html +autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev-html: - $(MAKE) html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0ef7d015be9b93..9dc9ba61e7a60f 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1248,19 +1248,24 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``1``, enable support for the Linux ``perf`` profiler. - Set by :option:`-X perf <-X>` command-line option and by the - :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support - with stack pointers and :option:`-X perf_jit <-X>` command-line option - and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf - support with DWARF JIT information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. + + Set to ``1`` by :option:`-X perf <-X>` command-line option and the + :envvar:`PYTHONPERFSUPPORT` environment variable. + + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 .. c:member:: int use_environment diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 188eec4592a270..e0ae0f77a01db9 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -159,7 +159,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 -.. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -181,6 +180,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 30e26fe52178d7..b2ac0c903c2bd7 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -317,7 +317,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -338,6 +338,8 @@ APIs: This is the recommended way to allocate a new Unicode object. Objects created using this function are not resizable. + On error, set an exception and return ``NULL``. + .. versionadded:: 3.3 @@ -614,6 +616,8 @@ APIs: Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -657,6 +661,8 @@ APIs: not out of bounds, and that the object can be modified safely (i.e. that it its reference count is one). + Return ``0`` on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -666,6 +672,8 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -674,6 +682,7 @@ APIs: Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 @@ -990,6 +999,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -1002,6 +1014,9 @@ These are the UTF-8 codec APIs: On error, set an exception, set *size* to ``-1`` (if it's not NULL) and return ``NULL``. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not responsible for deallocating the buffer. The buffer is deallocated and @@ -1429,8 +1444,9 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Compare a Unicode object with a char buffer which is interpreted as being UTF-8 or ASCII encoded and return true (``1``) if they are equal, or false (``0``) otherwise. - If the Unicode object contains surrogate characters or - the C string is not valid UTF-8, false (``0``) is returned. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. This function does not raise exceptions. diff --git a/Doc/conf.py b/Doc/conf.py index 27cf03d6bea05a..5f22340ac434c9 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -413,8 +413,8 @@ \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', - # The paper size ('letter' or 'a4'). - 'papersize': 'a4', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', } diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 446cc63cb34ff9..fc2ef33de5e5cc 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -18,6 +18,14 @@ Pending Removal in Python 3.16 Use the ``'w'`` format code (:c:type:`Py_UCS4`) for Unicode characters instead. +* :mod:`asyncio`: + + * :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`shutil`: * The :class:`!ExecError` exception diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 4a6f1ca57d89e3..fa7b22bde1dc6f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:attr:`self.__dict__ ` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ ` directly. How do I call a method defined in a base class from a derived class that extends it? diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index b98f91e023bdfc..70c34cde8a0659 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -47,11 +47,14 @@ lists. In contrast, the :func:`sorted` function accepts any iterable. Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making +The :meth:`list.sort` method and the functions :func:`sorted`, +:func:`min`, :func:`max`, :func:`heapq.nsmallest`, and +:func:`heapq.nlargest` have a *key* parameter to specify a function (or +other callable) to be called on each list element prior to making comparisons. -For example, here's a case-insensitive string comparison: +For example, here's a case-insensitive string comparison using +:meth:`str.casefold`: .. doctest:: @@ -272,6 +275,70 @@ to make it usable as a key function:: sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order +Strategies For Unorderable Types and Values +=========================================== + +A number of type and value issues can arise when sorting. +Here are some strategies that can help: + +* Convert non-comparable input types to strings prior to sorting: + +.. doctest:: + + >>> data = ['twelve', '11', 10] + >>> sorted(map(str, data)) + ['10', '11', 'twelve'] + +This is needed because most cross-type comparisons raise a +:exc:`TypeError`. + +* Remove special values prior to sorting: + +.. doctest:: + + >>> from math import isnan + >>> from itertools import filterfalse + >>> data = [3.3, float('nan'), 1.1, 2.2] + >>> sorted(filterfalse(isnan, data)) + [1.1, 2.2, 3.3] + +This is needed because the `IEEE-754 standard +`_ specifies that, "Every NaN +shall compare unordered with everything, including itself." + +Likewise, ``None`` can be stripped from datasets as well: + +.. doctest:: + + >>> data = [3.3, None, 1.1, 2.2] + >>> sorted(x for x in data if x is not None) + [1.1, 2.2, 3.3] + +This is needed because ``None`` is not comparable to other types. + +* Convert mapping types into sorted item lists before sorting: + +.. doctest:: + + >>> data = [{'a': 1}, {'b': 2}] + >>> sorted(data, key=lambda d: sorted(d.items())) + [{'a': 1}, {'b': 2}] + +This is needed because dict-to-dict comparisons raise a +:exc:`TypeError`. + +* Convert set types into sorted lists before sorting: + +.. doctest:: + + >>> data = [{'a', 'b', 'c'}, {'b', 'c', 'd'}] + >>> sorted(map(sorted, data)) + [['a', 'b', 'c'], ['b', 'c', 'd']] + +This is needed because the elements contained in set types do not have a +deterministic order. For example, ``list({'a', 'b'})`` may produce +either ``['a', 'b']`` or ``['b', 'a']``. + Odds and Ends ============= diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 2219e37f6b0677..37490456d13312 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -32,7 +32,7 @@ This module supports retrieving annotations in three main formats for annotations that cannot be resolved, allowing you to inspect the annotations without evaluating them. This is useful when you need to work with annotations that may contain unresolved forward references. -* :attr:`~Format.SOURCE` returns the annotations as a string, similar +* :attr:`~Format.STRING` returns the annotations as a string, similar to how it would appear in the source file. This is useful for documentation generators that want to display annotations in a readable way. @@ -135,7 +135,7 @@ Classes values. Real objects may contain references to, :class:`ForwardRef` proxy objects. - .. attribute:: SOURCE + .. attribute:: STRING :value: 3 Values are the text string of the annotation as it appears in the @@ -197,23 +197,23 @@ Classes Functions --------- -.. function:: annotations_to_source(annotations) +.. function:: annotations_to_string(annotations) Convert an annotations dict containing runtime values to a dict containing only strings. If the values are not already strings, - they are converted using :func:`value_to_source`. + they are converted using :func:`value_to_string`. This is meant as a helper for user-provided - annotate functions that support the :attr:`~Format.SOURCE` format but + annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. - For example, this is used to implement the :attr:`~Format.SOURCE` for + For example, this is used to implement the :attr:`~Format.STRING` for :class:`typing.TypedDict` classes created through the functional syntax: .. doctest:: >>> from typing import TypedDict >>> Movie = TypedDict("movie", {"name": str, "year": int}) - >>> get_annotations(Movie, format=Format.SOURCE) + >>> get_annotations(Movie, format=Format.STRING) {'name': 'str', 'year': 'int'} .. versionadded:: 3.14 @@ -282,7 +282,7 @@ Functions NameError: name 'undefined' is not defined >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) ForwardRef('undefined') - >>> call_evaluate_function(Alias.evaluate_value, Format.SOURCE) + >>> call_evaluate_function(Alias.evaluate_value, Format.STRING) 'undefined' .. versionadded:: 3.14 @@ -369,14 +369,14 @@ Functions .. versionadded:: 3.14 -.. function:: value_to_source(value) +.. function:: value_to_string(value) Convert an arbitrary Python value to a format suitable for use by the - :attr:`~Format.SOURCE` format. This calls :func:`repr` for most + :attr:`~Format.STRING` format. This calls :func:`repr` for most objects, but has special handling for some objects, such as type objects. This is meant as a helper for user-provided - annotate functions that support the :attr:`~Format.SOURCE` format but + annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. It can also be used to provide a user-friendly string representation for other objects that contain values that are commonly encountered in annotations. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d5a21899ae4f99..83d0a9ed7b1d0a 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -30,7 +30,7 @@ Quick Links for ArgumentParser ========================= =========================================================================================================== ================================================================================== Name Description Values ========================= =========================================================================================================== ================================================================================== -prog_ The name of the program Defaults to ``os.path.basename(sys.argv[0])`` +prog_ The name of the program usage_ The string describing the program usage description_ A brief description of what the program does epilog_ Additional description of the program after the argument help @@ -214,8 +214,8 @@ ArgumentParser objects as keyword arguments. Each parameter has its own more detailed description below, but in short they are: - * prog_ - The name of the program (default: - ``os.path.basename(sys.argv[0])``) + * prog_ - The name of the program (default: generated from the ``__main__`` + module attributes and ``sys.argv[0]``) * usage_ - The string describing the program usage (default: generated from arguments added to parser) @@ -268,10 +268,18 @@ The following sections describe how each of these are used. prog ^^^^ -By default, :class:`ArgumentParser` objects use the base name -(see :func:`os.path.basename`) of ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match the name that was +By default, :class:`ArgumentParser` calculates the name of the program +to display in help messages depending on the way the Python inerpreter was run: + +* The :func:`base name ` of ``sys.argv[0]`` if a file was + passed as argument. +* The Python interpreter name followed by ``sys.argv[0]`` if a directory or + a zipfile was passed as argument. +* The Python interpreter name followed by ``-m`` followed by the + module or package name if the :option:`-m` option was used. + +This default is almost +always desirable because it will make the help messages match the string that was used to invoke the program on the command line. For example, consider a file named ``myprogram.py`` with the following code:: @@ -281,7 +289,7 @@ named ``myprogram.py`` with the following code:: args = parser.parse_args() The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from): +(regardless of where the program was invoked from) if it is run as a script: .. code-block:: shell-session @@ -299,6 +307,17 @@ The help for this program will display ``myprogram.py`` as the program name -h, --help show this help message and exit --foo FOO foo help +If it is executed via the :option:`-m` option, the help will display a corresponding command line: + +.. code-block:: shell-session + + $ /usr/bin/python3 -m subdir.myprogram --help + usage: python3 -m subdir.myprogram [-h] [--foo FOO] + + options: + -h, --help show this help message and exit + --foo FOO foo help + To change this default behavior, another value can be supplied using the ``prog=`` argument to :class:`ArgumentParser`:: @@ -309,7 +328,8 @@ To change this default behavior, another value can be supplied using the options: -h, --help show this help message and exit -Note that the program name, whether determined from ``sys.argv[0]`` or from the +Note that the program name, whether determined from ``sys.argv[0]``, +from the ``__main__`` module attributes or from the ``prog=`` argument, is available to help messages using the ``%(prog)s`` format specifier. @@ -324,6 +344,9 @@ specifier. -h, --help show this help message and exit --foo FOO foo of the myprogram program +.. versionchanged:: 3.14 + The default ``prog`` value now reflects how ``__main__`` was actually executed, + rather than always being ``os.path.basename(sys.argv[0])``. usage ^^^^^ @@ -1823,7 +1846,7 @@ Sub-commands >>> >>> # create the parser for the "b" command >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') + >>> parser_b.add_argument('--baz', choices=('X', 'Y', 'Z'), help='baz help') >>> >>> # parse some argument lists >>> parser.parse_args(['a', '12']) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index a8a18ad31fb773..a9518859b83478 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -178,9 +178,9 @@ Root nodes A Python module, as with :ref:`file input `. Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - *body* is a :class:`list` of the module's :ref:`ast-statements`. + ``body`` is a :class:`list` of the module's :ref:`ast-statements`. - *type_ignores* is a :class:`list` of the module's type ignore comments; + ``type_ignores`` is a :class:`list` of the module's type ignore comments; see :func:`ast.parse` for more details. .. doctest:: @@ -199,7 +199,7 @@ Root nodes A single Python :ref:`expression input `. Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - *body* is a single node, + ``body`` is a single node, one of the :ref:`expression types `. .. doctest:: @@ -214,7 +214,7 @@ Root nodes A single :ref:`interactive input `, like in :ref:`tut-interac`. Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - *body* is a :class:`list` of :ref:`statement nodes `. + ``body`` is a :class:`list` of :ref:`statement nodes `. .. doctest:: @@ -243,9 +243,9 @@ Root nodes # type: (int, int) -> int return a + b - *argtypes* is a :class:`list` of :ref:`expression nodes `. + ``argtypes`` is a :class:`list` of :ref:`expression nodes `. - *returns* is a single :ref:`expression node `. + ``returns`` is a single :ref:`expression node `. .. doctest:: @@ -1771,9 +1771,9 @@ aliases. .. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. *name* is the name of the type variable. - *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. *default_value* + A :class:`typing.TypeVar`. ``name`` is the name of the type variable. + ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. ``default_value`` is the default value; if the :class:`!TypeVar` has no default, this attribute will be set to ``None``. @@ -1801,8 +1801,8 @@ aliases. .. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. - *default_value* is the default value; if the :class:`!ParamSpec` has no default, + A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + ``default_value`` is the default value; if the :class:`!ParamSpec` has no default, this attribute will be set to ``None``. .. doctest:: @@ -1836,8 +1836,8 @@ aliases. .. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. - *default_value* is the default value; if the :class:`!TypeVarTuple` has no + A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + ``default_value`` is the default value; if the :class:`!TypeVarTuple` has no default, this attribute will be set to ``None``. .. doctest:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index e3b24451188cc4..ce72127127c7a6 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -286,14 +286,6 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. - .. note:: - The default :mod:`multiprocessing` start method - (see :ref:`multiprocessing-start-methods`) will change away from - *fork* in Python 3.14. Code that requires *fork* be used for their - :class:`ProcessPoolExecutor` should explicitly specify that by - passing a ``mp_context=multiprocessing.get_context("fork")`` - parameter. - .. versionchanged:: 3.11 The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. @@ -310,6 +302,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *max_workers* uses :func:`os.process_cpu_count` by default, instead of :func:`os.cpu_count`. + .. versionchanged:: 3.14 + The default process start method (see + :ref:`multiprocessing-start-methods`) changed away from *fork*. If you + require the *fork* start method for :class:`ProcessPoolExecutor` you must + explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. _processpoolexecutor-example: ProcessPoolExecutor Example diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 1457392ce6e86c..e34b2db0210960 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -231,7 +231,7 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that @@ -300,6 +300,10 @@ Module contents .. versionadded:: 3.10 + - ``doc``: optional docstring for this field. + + .. versionadded:: 3.13 + If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not @@ -395,7 +399,7 @@ Module contents :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) Creates a new dataclass with name *cls_name*, fields as defined in *fields*, base classes as given in *bases*, and initialized @@ -411,6 +415,11 @@ Module contents of the dataclass is set to that value. By default, it is set to the module name of the caller. + The *decorator* parameter is a callable that will be used to create the dataclass. + It should take the class object as a first argument and the same keyword arguments + as :func:`@dataclass `. By default, the :func:`@dataclass ` + function is used. + This function is not strictly required, because any Python mechanism for creating a new class with :attr:`!__annotations__` can then apply the :func:`@dataclass ` function to convert that class to @@ -434,6 +443,9 @@ Module contents def add_one(self): return self.x + 1 + .. versionadded:: 3.14 + Added the *decorator* parameter. + .. function:: replace(obj, /, **changes) Creates a new object of the same type as *obj*, replacing diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 59e2dbd6847538..64510a77c67c11 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -295,6 +295,20 @@ Instance attributes (read-only): Between 0 and 86,399 inclusive. + .. caution:: + + It is a somewhat common bug for code to unintentionally use this attribute + when it is actually intended to get a :meth:`~timedelta.total_seconds` + value instead: + + .. doctest:: + + >>> from datetime import timedelta + >>> duration = timedelta(seconds=11235813) + >>> duration.days, duration.seconds + (130, 3813) + >>> duration.total_seconds() + 11235813.0 .. attribute:: timedelta.microseconds @@ -351,7 +365,7 @@ Supported operations: | | same value. (2) | +--------------------------------+-----------------------------------------------+ | ``-t1`` | Equivalent to ``timedelta(-t1.days, | -| | -t1.seconds*, -t1.microseconds)``, | +| | -t1.seconds, -t1.microseconds)``, | | | and to ``t1 * -1``. (1)(4) | +--------------------------------+-----------------------------------------------+ | ``abs(t)`` | Equivalent to ``+t`` when ``t.days >= 0``, | diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 774b3262117723..46136def06dc05 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -354,7 +354,7 @@ The :mod:`functools` module defines the following functions: newfunc.keywords = keywords return newfunc - The :func:`partial` function is used for partial function application which "freezes" + The :func:`!partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument @@ -368,10 +368,11 @@ The :mod:`functools` module defines the following functions: 18 If :data:`Placeholder` sentinels are present in *args*, they will be filled first - when :func:`partial` is called. This allows custom selection of positional arguments - to be pre-filled when constructing a :ref:`partial object `. + when :func:`!partial` is called. This makes it possible to pre-fill any positional + argument with a call to :func:`!partial`; without :data:`!Placeholder`, only the + first positional argument can be pre-filled. - If :data:`!Placeholder` sentinels are present, all of them must be filled at call time: + If any :data:`!Placeholder` sentinels are present, all must be filled at call time: .. doctest:: @@ -379,14 +380,15 @@ The :mod:`functools` module defines the following functions: >>> say_to_world('Hello', 'dear') Hello dear world! - Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`, because - only one positional argument is provided, while there are two placeholders - in :ref:`partial object `. + Calling ``say_to_world('Hello')`` raises a :exc:`TypeError`, because + only one positional argument is provided, but there are two placeholders + that must be filled in. - Successive :func:`partial` applications fill :data:`!Placeholder` sentinels - of the input :func:`partial` objects with new positional arguments. - A place for positional argument can be retained by inserting new - :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`: + If :func:`!partial` is applied to an existing :func:`!partial` object, + :data:`!Placeholder` sentinels of the input object are filled in with + new positional arguments. + A placeholder can be retained by inserting a new + :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: .. doctest:: @@ -402,8 +404,8 @@ The :mod:`functools` module defines the following functions: >>> remove_first_dear(message) 'Hello, dear world!' - Note, :data:`!Placeholder` has no special treatment when used for keyword - argument of :data:`!Placeholder`. + :data:`!Placeholder` has no special treatment when used in a keyword + argument to :func:`!partial`. .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. @@ -541,6 +543,25 @@ The :mod:`functools` module defines the following functions: ... print(arg.real, arg.imag) ... + For code that dispatches on a collections type (e.g., ``list``), but wants + to typehint the items of the collection (e.g., ``list[int]``), the + dispatch type should be passed explicitly to the decorator itself with the + typehint going into the function definition:: + + >>> @fun.register(list) + ... def _(arg: list[int], verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + .. note:: + + At runtime the function will dispatch on an instance of a list regardless + of the type contained within the list i.e. ``[1,2,3]`` will be + dispatched the same as ``["foo", "bar", "baz"]``. The annotation + provided in this example is for static type checkers only and has no + runtime impact. To enable registering :term:`lambdas` and pre-existing functions, the :func:`register` attribute can also be used in a functional form:: @@ -791,7 +812,7 @@ have three read-only attributes: The keyword arguments that will be supplied when the :class:`partial` object is called. -:class:`partial` objects are like :class:`function` objects in that they are +:class:`partial` objects are like :ref:`function objects ` in that they are callable, weak referenceable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes +differences. For instance, the :attr:`~definition.__name__` and :attr:`~definition.__doc__` attributes are not created automatically. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index e4cef1f3e3b7c0..27d31f66b12495 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1166,10 +1166,9 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's :attr:`__spec__` attribute. In the - descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object, - e.g. ``module.__spec__.origin == module.__file__``. Note, however, that + typically exposed as the module's :attr:`__spec__` attribute. Many + of these attributes are also available directly on a module: for example, + ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is no synchronization between the two objects. For example, it is possible to update the module's :attr:`__file__` at runtime and this will not be automatically @@ -1179,66 +1178,60 @@ find and load modules. .. attribute:: name - (:attr:`__name__`) - - The module's fully qualified name. - The :term:`finder` should always set this attribute to a non-empty string. + The module's fully qualified name + (see :attr:`__name__` attributes on modules). + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (:attr:`__loader__`) - - The :term:`loader` used to load the module. - The :term:`finder` should always set this attribute. + The :term:`loader` used to load the module + (see :attr:`__loader__` attributes on modules). + The :term:`finder` should always set this attribute. .. attribute:: origin - (:attr:`__file__`) - - The location the :term:`loader` should use to load the module. - For example, for modules loaded from a .py file this is the filename. - The :term:`finder` should always set this attribute to a meaningful value - for the :term:`loader` to use. In the uncommon case that there is not one - (like for namespace packages), it should be set to ``None``. + The location the :term:`loader` should use to load the module + (see :attr:`__file__` attributes on modules). + For example, for modules loaded from a .py file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (:attr:`__path__`) - - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The :term:`finder` should set this attribute to a list, even an empty one, to indicate - to the import system that the module is a package. It should be set to ``None`` for - non-package modules. It is set automatically later to a special object for - namespace packages. + The list of locations where the package's submodules will be found + (see :attr:`__path__` attributes on modules). + Most of the time this is a single directory. + The :term:`finder` should set this attribute to a list, even an empty one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - The :term:`finder` may set this attribute to an object containing additional, - module-specific data to use when loading the module. Otherwise it should be - set to ``None``. + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (:attr:`__cached__`) - - The filename of a compiled version of the module's code. - The :term:`finder` should always set this attribute but it may be ``None`` - for modules that do not need compiled code stored. + The filename of a compiled version of the module's code + (see :attr:`__cached__` attributes on modules). + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (:attr:`__package__`) - - (Read-only) The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`name`. + (Read-only) The fully qualified name of the package the module is in (or the + empty string for a top-level module). + See :attr:`__package__` attributes on modules. + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`origin` is interpreted + and how the module's :attr:`__file__` is populated. .. class:: AppleFrameworkLoader(name, path) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c1299ebfe8d27a..9a62249816c9bf 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -58,7 +58,7 @@ Iterator Arguments Results :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','ABC'], len) → (1, A B) (3, ABC)`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` @@ -93,7 +93,7 @@ Examples Results Itertool Functions ------------------ -The following module functions all construct and return iterators. Some provide +The following functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. @@ -131,11 +131,12 @@ loops that truncate the stream. total = function(total, element) yield total - The *function* argument can be set to :func:`min` for a running - minimum, :func:`max` for a running maximum, or :func:`operator.mul` - for a running product. `Amortization tables - `_ - can be built by accumulating interest and applying payments: + To compute a running minimum, set *function* to :func:`min`. + For a running maximum, set *function* to :func:`max`. + Or for a running product, set *function* to :func:`operator.mul`. + To build an `Amortization table + `_, + accumulate the interest and apply payments: .. doctest:: @@ -202,10 +203,10 @@ loops that truncate the stream. .. function:: chain(*iterables) - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. This combines multiple data sources into a + single iterator. Roughly equivalent to:: def chain(*iterables): # chain('ABC', 'DEF') → A B C D E F @@ -353,10 +354,12 @@ loops that truncate the stream. def cycle(iterable): # cycle('ABCD') → A B C D A B C D A B C D ... + saved = [] for element in iterable: yield element saved.append(element) + while saved: for element in saved: yield element @@ -396,8 +399,10 @@ loops that truncate the stream. def filterfalse(predicate, iterable): # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 + if predicate is None: predicate = bool + for x in iterable: if not predicate(x): yield x @@ -474,7 +479,7 @@ loops that truncate the stream. If *start* is zero or ``None``, iteration starts at zero. Otherwise, elements from the iterable are skipped until *start* is reached. - If *stop* is ``None``, iteration continues until the iterable is + If *stop* is ``None``, iteration continues until the input is exhausted, if at all. Otherwise, it stops at the specified position. If *step* is ``None``, the step defaults to one. Elements are returned @@ -520,8 +525,10 @@ loops that truncate the stream. def pairwise(iterable): # pairwise('ABCDEFG') → AB BC CD DE EF FG + iterator = iter(iterable) a = next(iterator, None) + for b in iterator: yield a, b a = b @@ -584,7 +591,8 @@ loops that truncate the stream. .. function:: product(*iterables, repeat=1) - Cartesian product of input iterables. + `Cartesian product `_ + of the input iterables. Roughly equivalent to nested for-loops in a generator expression. For example, ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 80d6e4dae24463..036b8f44b9ff3b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -124,11 +124,11 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX except macOS. + Available on POSIX systems. - .. note:: - The default start method will change away from *fork* in Python 3.14. - Code that requires *fork* should explicitly specify that via + .. versionchanged:: 3.14 + This is no longer the default start method on any platform. + Code that requires *fork* must explicitly specify that via :func:`get_context` or :func:`set_start_method`. .. versionchanged:: 3.12 @@ -146,9 +146,11 @@ to start a process. These *start methods* are side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on POSIX platforms which support passing file descriptors - over Unix pipes such as Linux. + Available on POSIX platforms which support passing file descriptors over + Unix pipes such as Linux. The default on those. + .. versionchanged:: 3.14 + This became the default start method on POSIX platforms. .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for @@ -162,6 +164,13 @@ to start a process. These *start methods* are method should be considered unsafe as it can lead to crashes of the subprocess as macOS system libraries may start threads. See :issue:`33725`. +.. versionchanged:: 3.14 + + On POSIX platforms the default start method was changed from *fork* to + *forkserver* to retain the performance but avoid common multithreaded + process incompatibilities. See :gh:`84559`. + + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 1682eb0fbea42d..6c099b22b38c21 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -439,17 +439,20 @@ can be overridden by the local file. Specifying any command resuming execution (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, - :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) + :pdbcmd:`return`, :pdbcmd:`until`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint—which could have its own command list, leading to ambiguities about which list to execute. - If you use the ``silent`` command in the command list, the usual message about - stopping at a breakpoint is not printed. This may be desirable for breakpoints - that are to print a specific message and then continue. If none of the other - commands print anything, you see no sign that the breakpoint was reached. + If the list of commands contains the ``silent`` command, or a command that + resumes execution, then the breakpoint message containing information about + the frame is not displayed. + + .. versionchanged:: 3.14 + Frame information will not be displayed if a command that resumes execution + is present in the command list. .. pdbcommand:: s(tep) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1f316307965c11..57a1f920523035 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -574,11 +574,13 @@ The available presentation types for :class:`float` and | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | + | None | For :class:`float` this is like the ``'g'`` type, except | | | that when fixed-point notation is used to format the | | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | + | | decimal point, and switches to the scientific notation | + | | when ``exp >= p - 1``. When the precision is not | + | | specified, the latter will be as large as needed to | + | | represent the given value faithfully. | | | | | | For :class:`~decimal.Decimal`, this is the same as | | | either ``'g'`` or ``'G'`` depending on the value of | diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index ac8bcceaca5aeb..f7140af2494898 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -50,16 +50,14 @@ Registering and using tools *tool_id* must be in the range 0 to 5 inclusive. Raises a :exc:`ValueError` if *tool_id* is in use. -.. function:: free_tool_id(tool_id: int, /) -> None +.. function:: clear_tool_id(tool_id: int, /) -> None - Should be called once a tool no longer requires *tool_id*. + Unregister all events and callback functions associated with *tool_id*. -.. note:: +.. function:: free_tool_id(tool_id: int, /) -> None - :func:`free_tool_id` will not disable global or local events associated - with *tool_id*, nor will it unregister any callback functions. This - function is only intended to be used to notify the VM that the - particular *tool_id* is no longer in use. + Should be called once a tool no longer requires *tool_id*. + Will call :func:`clear_tool_id` before releasing *tool_id*. .. function:: get_tool(tool_id: int, /) -> str | None diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c08b10d67bb031..cd8b90854b0e94 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2349,7 +2349,9 @@ types. Backward-compatible usage:: - # For creating a generic NamedTuple on Python 3.11 or lower + # For creating a generic NamedTuple on Python 3.11 + T = TypeVar("T") + class Group(NamedTuple, Generic[T]): key: T group: list[T] @@ -3427,7 +3429,7 @@ Introspection helpers * Replaces type hints that evaluate to :const:`!None` with :class:`types.NoneType`. * Supports the :attr:`~annotationlib.Format.FORWARDREF` and - :attr:`~annotationlib.Format.SOURCE` formats. + :attr:`~annotationlib.Format.STRING` formats. *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. *owner*, if given, should be the object that holds the annotations that diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index d603a163c2e82d..cc2b1b4299553c 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -401,6 +401,8 @@ the *new_callable* argument to :func:`patch`. The reset_mock method resets all the call attributes on a mock object: + .. doctest:: + >>> mock = Mock(return_value=None) >>> mock('hello') >>> mock.called @@ -409,20 +411,41 @@ the *new_callable* argument to :func:`patch`. >>> mock.called False - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the + reuse the same object. + + *return_value* parameter when set to ``True`` resets :attr:`return_value`: + + .. doctest:: + + >>> mock = Mock(return_value=5) + >>> mock('hello') + 5 + >>> mock.reset_mock(return_value=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + *side_effect* parameter when set to ``True`` resets :attr:`side_effect`: + + .. doctest:: + + >>> mock = Mock(side_effect=ValueError) + >>> mock('hello') + Traceback (most recent call last): + ... + ValueError + >>> mock.reset_mock(side_effect=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + Note that :meth:`reset_mock` *doesn't* clear the :attr:`return_value`, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - :attr:`return_value` or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. + set using normal assignment by default. - .. note:: *return_value*, and *side_effect* are keyword-only - arguments. + Child mocks are reset as well. + .. versionchanged:: 3.6 + Added two keyword-only arguments to the reset_mock function. .. method:: mock_add_spec(spec, spec_set=False) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index cf6c5437be4fd1..e2c77963ff3040 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -37,14 +37,14 @@ A virtual environment is (amongst other things): are by default isolated from software in other virtual environments and Python interpreters and libraries installed in the operating system. -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in +* Contained in a directory, conventionally named ``.venv`` or ``venv`` in the project directory, or under a container directory for lots of virtual environments, such as ``~/.virtualenvs``. * Not checked into source control systems such as Git. * Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment + scratch. You don't place any project code in the environment. * Not considered as movable or copyable -- you just recreate the same environment in the target location. @@ -61,7 +61,127 @@ See :pep:`405` for more background on Python virtual environments. Creating virtual environments ----------------------------- -.. include:: /using/venv-create.inc +:ref:`Virtual environments ` are created by executing the ``venv`` +module: + +.. code-block:: shell + + python -m venv /path/to/new/virtual/environment + +This creates the target directory (including parent directories as needed) +and places a :file:`pyvenv.cfg` file in it with a ``home`` key +pointing to the Python installation from which the command was run. +It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory +containing a copy or symlink of the Python executable +(as appropriate for the platform or arguments used at environment creation time). +It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory +(on Windows, this is :file:`Lib\site-packages`). +If an existing directory is specified, it will be re-used. + +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + +.. deprecated-removed:: 3.6 3.8 + :program:`pyvenv` was the recommended tool for creating virtual environments + for Python 3.3 and 3.4, and replaced in 3.5 by executing ``venv`` directly. + +.. highlight:: none + +On Windows, invoke the ``venv`` command as follows: + +.. code-block:: ps1con + + PS> python -m venv C:\path\to\new\virtual\environment + +The command, if run with ``-h``, will show the available options:: + + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-files] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory + if it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been + upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-files + Skips adding SCM ignore files to the environment + directory (Git is supported by default). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options. + +.. versionchanged:: 3.4 + In earlier versions, if the target directory already existed, an error was + raised, unless the ``--clear`` or ``--upgrade`` option was provided. + +.. versionchanged:: 3.9 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI. + +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + +.. versionchanged:: 3.13 + + Added the ``--without-scm-ignore-files`` option. +.. versionchanged:: 3.13 + ``venv`` now creates a :file:`.gitignore` file for Git by default. + +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + +.. note:: + On Microsoft Windows, it may be required to enable the ``Activate.ps1`` + script by setting the execution policy for the user. You can do this by + issuing the following PowerShell command: + + .. code-block:: powershell + + PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + + See `About Execution Policies + `_ + for more information. + +The created :file:`pyvenv.cfg` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + +Multiple paths can be given to ``venv``, in which case an identical virtual +environment will be created, according to the given options, at each provided +path. .. _venv-explanation: @@ -117,7 +237,7 @@ should be runnable without activating it. In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{}/bin/python`. +:samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed @@ -168,31 +288,31 @@ creation according to their needs, the :class:`EnvBuilder` class. The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: - * ``system_site_packages`` -- a Boolean value indicating that the system Python + * *system_site_packages* -- a boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete the contents of + * *clear* -- a boolean value which, if true, will delete the contents of any existing target directory, before creating the environment. - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + * *symlinks* -- a boolean value indicating whether to attempt to symlink the Python binary rather than copying. - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + * *upgrade* -- a boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). - * ``with_pip`` -- a Boolean value which, if true, ensures pip is + * *with_pip* -- a boolean value which, if true, ensures pip is installed in the virtual environment. This uses :mod:`ensurepip` with the ``--default-pip`` option. - * ``prompt`` -- a String to be used after virtual environment is activated + * *prompt* -- a string to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would be used). If the special string ``"."`` is provided, the basename of the current directory is used as the prompt. - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * *upgrade_deps* -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_files`` -- Create ignore files based for the specified source + * *scm_ignore_files* -- Create ignore files based for the specified source control managers (SCM) in the iterable. Support is defined by having a method named ``create_{scm}_ignore_file``. The only value supported by default is ``"git"`` via :meth:`create_git_ignore_file`. @@ -210,10 +330,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: + :class:`EnvBuilder` may be used as a base class. .. method:: create(env_dir) @@ -313,14 +430,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip``) + Upgrades the core venv dependency packages (currently :pypi:`pip`) in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 .. versionchanged:: 3.12 - ``setuptools`` is no longer a core venv dependency. + :pypi:`setuptools` is no longer a core venv dependency. .. method:: post_setup(context) @@ -328,25 +445,15 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - .. method:: install_scripts(context, path) + This method can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the + ``common``, ``posix``, ``nt``; each containing scripts destined for the + ``bin`` directory in the environment. The contents of ``common`` and the directory corresponding to :data:`os.name` are copied after some text replacement of placeholders: @@ -371,10 +478,20 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: create_git_ignore_file(context) Creates a ``.gitignore`` file within the virtual environment that causes - the entire directory to be ignored by the ``git`` source control manager. + the entire directory to be ignored by the Git source control manager. .. versionadded:: 3.13 + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ @@ -387,16 +504,16 @@ There is also a module-level convenience function: .. versionadded:: 3.3 .. versionchanged:: 3.4 - Added the ``with_pip`` parameter + Added the *with_pip* parameter .. versionchanged:: 3.6 - Added the ``prompt`` parameter + Added the *prompt* parameter .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter + Added the *upgrade_deps* parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_files`` parameter + Added the *scm_ignore_files* parameter An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 097a39cea31c67..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1217,9 +1217,10 @@ A function definition defines a user-defined function object (see section : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + parameter_list_starargs: "*" [`star_parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] + star_parameter: `identifier` [":" ["*"] `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -1326,11 +1327,16 @@ and may only be passed by positional arguments. Parameters may have an :term:`annotation ` of the form "``: expression``" following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of +``*identifier`` or ``**identifier``. (As a special case, parameters of the form +``*identifier`` may have an annotation "``: *expression``".) Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the semantics of a function. See :ref:`annotations` for more information on annotations. +.. versionchanged:: 3.11 + Parameters of the form "``*identifier``" may have an annotation + "``: *expression``". See :pep:`646`. + .. index:: pair: lambda; expression It is also possible to create anonymous functions (functions not bound to a diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 5ce6bf17db41ea..513199d21456bf 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1080,7 +1080,10 @@ Special attributes .. versionadded:: 3.13 * - .. attribute:: type.__firstlineno__ - - The line number of the first line of the class definition, including decorators. + - The line number of the first line of the class definition, + including decorators. + Setting the :attr:`__module__` attribute removes the + :attr:`!__firstlineno__` item from the type's dictionary. .. versionadded:: 3.13 diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b5f5523d368964..ab72ad49d041e1 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -284,7 +284,7 @@ A list display is a possibly empty series of expressions enclosed in square brackets: .. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" + list_display: "[" [`flexible_expression_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either a list of expressions or a comprehension. When a comma-separated list of @@ -309,7 +309,7 @@ A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: .. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" + set_display: "{" (`flexible_expression_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by either a sequence of expressions or a comprehension. When a comma-separated @@ -454,7 +454,7 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_from: "yield" "from" `expression` - yield_expression: "yield" `expression_list` | `yield_from` + yield_expression: "yield" `yield_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -485,9 +485,9 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` +suspended again, returning the value of :token:`~python-grammar:yield_list` to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. +or ``None`` if :token:`~python-grammar:yield_list` is omitted. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -576,7 +576,7 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + :token:`~python-grammar:yield_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -695,7 +695,7 @@ how a generator object would be used in a :keyword:`for` statement. Calling one of the asynchronous generator's methods returns an :term:`awaitable` object, and the execution starts when this object is awaited on. At that time, the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the +again, returning the value of :token:`~python-grammar:yield_list` to the awaiting coroutine. As with a generator, suspension means that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -759,7 +759,7 @@ which are used to control the execution of a generator function. asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the + expression. The value of the :token:`~python-grammar:yield_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a @@ -892,7 +892,7 @@ will generally select an element from the container. The subscription of a :ref:`GenericAlias ` object. .. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" + subscription: `primary` "[" `flexible_expression_list` "]" When an object is subscripted, the interpreter will evaluate the primary and the expression list. @@ -904,9 +904,13 @@ primary is subscripted, the evaluated result of the expression list will be passed to one of these methods. For more details on when ``__class_getitem__`` is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. +If the expression list contains at least one comma, or if any of the expressions +are starred, the expression list will evaluate to a :class:`tuple` containing +the items of the expression list. Otherwise, the expression list will evaluate +to the value of the list's sole member. + +.. versionchanged:: 3.11 + Expressions in an expression list may be starred. See :pep:`646`. For built-in objects, there are two types of objects that support subscription via :meth:`~object.__getitem__`: @@ -1803,6 +1807,7 @@ returns a boolean value regardless of the type of its argument single: assignment expression single: walrus operator single: named expression + pair: assignment; expression Assignment expressions ====================== @@ -1905,10 +1910,12 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar + starred_expression: ["*"] `or_expr` + flexible_expression: `assignment_expression` | `starred_expression` + flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] + starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` + yield_list: `expression_list` | `starred_expression` "," [`starred_expression_list`] .. index:: pair: object; tuple @@ -1929,6 +1936,9 @@ the unpacking. .. versionadded:: 3.5 Iterable unpacking in expression lists, originally proposed by :pep:`448`. +.. versionadded:: 3.11 + Any item in an expression list may be starred. See :pep:`646`. + .. index:: pair: trailing; comma A trailing comma is required only to create a one-item tuple, diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py index 53ea1bf47b8fd3..9ccaec3dd5ce0f 100644 --- a/Doc/tools/extensions/patchlevel.py +++ b/Doc/tools/extensions/patchlevel.py @@ -74,4 +74,8 @@ def get_version_info(): if __name__ == "__main__": - print(format_version_info(get_header_version_info())[0]) + short_ver, full_ver = format_version_info(get_header_version_info()) + if sys.argv[1:2] == ["--short"]: + print(short_ver) + else: + print(full_ver) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 1f725c2377035b..c89b1693343b4e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -288,6 +288,9 @@ def run(self): version_deprecated = expand_version_arg(self.arguments[0], self.config.release) version_removed = self.arguments.pop(1) + if version_removed == 'next': + raise ValueError( + 'deprecated-removed:: second argument cannot be `next`') self.arguments[0] = version_deprecated, version_removed # Set the label based on if we have reached the removal version diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index c978e61b16a49e..45ec436fee72d7 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -13,7 +13,7 @@ {% endif %} {% block body %} -

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

+

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index c97c65f7a3988e..fd765e58ff2485 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -209,8 +209,10 @@ after the loop finishes its final iteration, that is, if no break occurred. In a :keyword:`while` loop, it's executed after the loop's condition becomes false. -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. +In either kind of loop, the :keyword:`!else` clause is **not** executed if the +loop was terminated by a :keyword:`break`. Of course, other ways of ending the +loop early, such as a :keyword:`return` or a raised exception, will also skip +execution of the :keyword:`else` clause. This is exemplified in the following :keyword:`!for` loop, which searches for prime numbers:: diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 2dfac0758435d1..4b6c884f3d4f25 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -2,140 +2,223 @@ .. _using-on-mac: ********************* -Using Python on a Mac +Using Python on macOS ********************* -:Author: Bob Savage +.. sectionauthor:: Bob Savage +.. sectionauthor:: Ned Deily +This document aims to give an overview of macOS-specific behavior you should +know about to get started with Python on Mac computers. +Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, +but there are some differences in installation and some features. -Python on a Mac running macOS is in principle very similar to Python on -any other Unix platform, but there are a number of additional features such as -the integrated development environment (IDE) and the Package Manager that are -worth pointing out. +There are various ways to obtain and install Python for macOS. +Pre-built versions of the most recent versions of Python are available +from a number of distributors. Much of this document describes use of +the Pythons provided by the CPython release team for download from +the `python.org website `_. See +:ref:`alternative_bundles` for some other options. +.. |usemac_x_dot_y| replace:: 3.13 +.. |usemac_python_x_dot_y_literal| replace:: ``python3.13`` +.. |usemac_python_x_dot_y_t_literal| replace:: ``python3.13t`` +.. |usemac_python_x_dot_y_t_literal_config| replace:: ``python3.13t-config`` +.. |usemac_applications_folder_name| replace:: ``Python 3.13`` +.. |usemac_applications_folder_version| replace:: ``/Applications/Python 3.13/`` .. _getting-osx: .. _getting-and-installing-macpython: -Getting and Installing Python -============================= +Using Python for macOS from ``python.org`` +========================================== -macOS used to come with Python 2.7 pre-installed between versions -10.8 and `12.3 `_. -You are invited to install the most recent version of Python 3 from the `Python -website `__. -A current "universal2 binary" build of Python, which runs natively on the Mac's -new Apple Silicon and legacy Intel processors, is available there. +Installation steps +------------------ -What you get after installing is a number of things: +For `current Python versions `_ +(other than those in ``security`` status), the release team produces a +**Python for macOS** installer package for each new release. +A list of available installers +is available `here `_. +We recommend using the most recent supported Python version where possible. +Current installers provide a +`universal2 binary `_ build +of Python which runs natively on all Macs (Apple Silicon and Intel) that are +supported by a wide range of macOS versions, +currently typically from at least **macOS 10.13 High Sierra** on. -* A |python_version_literal| folder in your :file:`Applications` folder. In here - you find IDLE, the development environment that is a standard part of official +The downloaded file is a standard macOS installer package file (``.pkg``). +File integrity information (checksum, size, sigstore signature, etc) for each file is included +on the release download page. Installer packages and their contents are signed and notarized +with ``Python Software Foundation`` Apple Developer ID certificates +to meet `macOS Gatekeeper requirements `_. + +For a default installation, double-click on the downloaded installer package file. +This should launch the standard macOS Installer app and display the first of several +installer windows steps. + +.. image:: mac_installer_01_introduction.png + +Clicking on the **Continue** button brings up the **Read Me** for this installer. +Besides other important information, the **Read Me** documents which Python version is +going to be installed and on what versions of macOS it is supported. You may need +to scroll through to read the whole file. By default, this **Read Me** will also be +installed in |usemac_applications_folder_version| and available to read anytime. + +.. image:: mac_installer_02_readme.png + +Clicking on **Continue** proceeds to display the license for Python and for +other included software. You will then need to **Agree** to the license terms +before proceeding to the next step. This license file will also be installed +and available to be read later. + +.. image:: mac_installer_03_license.png + +After the license terms are accepted, the next step is the **Installation Type** +display. For most uses, the standard set of installation operations is appropriate. + +.. image:: mac_installer_04_installation_type.png + +By pressing the **Customize** button, you can choose to omit or select certain package +components of the installer. Click on each package name to see a description of +what it installs. +To also install support for the optional experimental free-threaded feature, +see :ref:`install-freethreaded-macos`. + +.. image:: mac_installer_05_custom_install.png + +In either case, clicking **Install** will begin the install process by asking +permission to install new software. A macOS user name with ``Administrator`` privilege +is needed as the installed Python will be available to all users of the Mac. + +When the installation is complete, the **Summary** window will appear. + +.. image:: mac_installer_06_summary.png + +Double-click on the :command:`Install Certificates.command` +icon or file in the |usemac_applications_folder_version| window to complete the +installation. + +.. image:: mac_installer_07_applications.png + +This will open a temporary :program:`Terminal` shell window that +will use the new Python to download and install SSL root certificates +for its use. + +.. image:: mac_installer_08_install_certificates.png + +If ``Successfully installed certifi`` and ``update complete`` appears +in the terminal window, the installation is complete. +Close this terminal window and the installer window. + +A default install will include: + +* A |usemac_applications_folder_name| folder in your :file:`Applications` folder. In here + you find :program:`IDLE`, the development environment that is a standard part of official Python distributions; and :program:`Python Launcher`, which handles double-clicking Python - scripts from the Finder. + scripts from the macOS `Finder `_. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell - path. To uninstall Python, you can remove these three things. A - symlink to the Python executable is placed in :file:`/usr/local/bin/`. + path. To uninstall Python, you can remove these three things. + Symlinks to the Python executable are placed in :file:`/usr/local/bin/`. .. note:: - On macOS 10.8-12.3, the Apple-provided build of Python is installed in - :file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, - respectively. You should never modify or delete these, as they are - Apple-controlled and are used by Apple- or third-party software. Remember that - if you choose to install a newer Python version from python.org, you will have - two different but functional Python installations on your computer, so it will - be important that your paths and usages are consistent with what you want to do. - -IDLE includes a Help menu that allows you to access Python documentation. If you -are completely new to Python you should start reading the tutorial introduction -in that document. - -If you are familiar with Python on other Unix platforms you should read the -section on running Python scripts from the Unix shell. - + Recent versions of macOS include a :command:`python3` command in :file:`/usr/bin/python3` + that links to a usually older and incomplete version of Python provided by and for use by + the Apple development tools, :program:`Xcode` or the :program:`Command Line Tools for Xcode`. + You should never modify or attempt to delete this installation, as it is + Apple-controlled and is used by Apple-provided or third-party software. If + you choose to install a newer Python version from ``python.org``, you will have + two different but functional Python installations on your computer that + can co-exist. The default installer options should ensure that its :command:`python3` + will be used instead of the system :command:`python3`. How to run a Python script -------------------------- -Your best way to get started with Python on macOS is through the IDLE -integrated development environment; see section :ref:`ide` and use the Help menu -when the IDE is running. +There are two ways to invoke the Python interpreter. +If you are familiar with using a Unix shell in a terminal +window, you can invoke |usemac_python_x_dot_y_literal| or ``python3`` optionally +followed by one or more command line options (described in :ref:`using-on-general`). +The Python tutorial also has a useful section on +:ref:`using Python interactively from a shell `. + +You can also invoke the interpreter through an integrated +development environment. +:ref:`idle` is a basic editor and interpreter environment +which is included with the standard distribution of Python. +:program:`IDLE` includes a Help menu that allows you to access Python documentation. If you +are completely new to Python, you can read the tutorial introduction +in that document. -If you want to run Python scripts from the Terminal window command line or from -the Finder you first need an editor to create your script. macOS comes with a -number of standard Unix command line editors, :program:`vim` -:program:`nano` among them. If you want a more Mac-like editor, -:program:`BBEdit` from Bare Bones Software (see -https://www.barebones.com/products/bbedit/index.html) are good choices, as is -:program:`TextMate` (see https://macromates.com). Other editors include -:program:`MacVim` (https://macvim.org) and :program:`Aquamacs` -(https://aquamacs.org). +There are many other editors and IDEs available, see :ref:`editors` +for more information. -To run your script from the Terminal window you must make sure that -:file:`/usr/local/bin` is in your shell search path. +To run a Python script file from the terminal window, you can +invoke the interpreter with the name of the script file: -To run your script from the Finder you have two options: + |usemac_python_x_dot_y_literal| ``myscript.py`` + +To run your script from the Finder, you can either: * Drag it to :program:`Python Launcher`. * Select :program:`Python Launcher` as the default application to open your - script (or any ``.py`` script) through the finder Info window and double-click it. + script (or any ``.py`` script) through the Finder Info window and double-click it. :program:`Python Launcher` has various preferences to control how your script is launched. Option-dragging allows you to change these for one invocation, or use - its Preferences menu to change things globally. - - -.. _osx-gui-scripts: - -Running scripts with a GUI --------------------------- - -With older versions of Python, there is one macOS quirk that you need to be -aware of: programs that talk to the Aqua window manager (in other words, -anything that has a GUI) need to be run in a special way. Use :program:`pythonw` -instead of :program:`python` to start such scripts. + its ``Preferences`` menu to change things globally. -With Python 3.9, you can use either :program:`python` or :program:`pythonw`. +Be aware that running the script directly from the macOS Finder might +produce different results than when running from a terminal window as +the script will not be run in the usual shell environment including +any setting of environment variables in shell profiles. +And, as with any other script or program, +be certain of what you are about to run. +.. _alternative_bundles: -Configuration -------------- +Alternative Distributions +========================= -Python on macOS honors all standard Unix environment variables such as -:envvar:`PYTHONPATH`, but setting these variables for programs started from the -Finder is non-standard as the Finder does not read your :file:`.profile` or -:file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's -`Technical Q&A QA1067 `__ -for details. +Besides the standard ``python.org`` for macOS installer, there are third-party +distributions for macOS that may include additional functionality. +Some popular distributions and their key features: -For more information on installation Python packages, see section -:ref:`mac-package-manager`. +`ActivePython `_ + Installer with multi-platform compatibility, documentation +`Anaconda `_ + Popular scientific modules (such as numpy, scipy, and pandas) and the + ``conda`` package manager. -.. _ide: +`Homebrew `_ + Package manager for macOS including multiple versions of Python and many + third-party Python-based packages (including numpy, scipy, and pandas). -The IDE -======= - -Python ships with the standard IDLE development environment. A good -introduction to using IDLE can be found at -https://www.hashcollision.org/hkn/python/idle_intro/index.html. +`MacPorts `_ + Another package manager for macOS including multiple versions of Python and many + third-party Python-based packages. May include pre-built versions of Python and + many packages for older versions of macOS. +Note that distributions might not include the latest versions of Python or +other libraries, and are not maintained or supported by the core Python team. .. _mac-package-manager: Installing Additional Python Packages ===================================== -This section has moved to the `Python Packaging User Guide`_. +Refer to the `Python Packaging User Guide`_ for more information. .. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ +.. _osx-gui-scripts: + .. _gui-programming-on-the-mac: GUI Programming @@ -143,36 +226,209 @@ GUI Programming There are several options for building GUI applications on the Mac with Python. -*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is -the foundation of most modern Mac development. Information on PyObjC is -available from :pypi:`pyobjc`. - The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform -Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with -macOS by Apple, and the latest version can be downloaded and installed from -https://www.activestate.com; it can also be built from source. +Tk toolkit (https://www.tcl.tk). A macOS-native version of Tk is included with +the installer. + +*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework. +Information on PyObjC is available from :pypi:`pyobjc`. -A number of alternative macOS GUI toolkits are available: +A number of alternative macOS GUI toolkits are available including: -* `PySide `__: Official Python bindings to the - `Qt GUI toolkit `__. +* `PySide `_: Official Python bindings to the + `Qt GUI toolkit `_. -* `PyQt `__: Alternative +* `PyQt `_: Alternative Python bindings to Qt. -* `Kivy `__: A cross-platform GUI toolkit that supports +* `Kivy `_: A cross-platform GUI toolkit that supports desktop and mobile platforms. -* `Toga `__: Part of the `BeeWare Project - `__; supports desktop, mobile, web and console apps. +* `Toga `_: Part of the `BeeWare Project + `_; supports desktop, mobile, web and console apps. -* `wxPython `__: A cross-platform toolkit that +* `wxPython `_: A cross-platform toolkit that supports desktop operating systems. + +Advanced Topics +=============== + +.. _install-freethreaded-macos: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +The ``python.org`` :ref:`Python for macOS ` +installer package can optionally install an additional build of +Python |usemac_x_dot_y| that supports :pep:`703`, the experimental free-threading feature +(running with the :term:`global interpreter lock` disabled). +Check the release page on ``python.org`` for possible updated information. + +Because this feature is still considered experimental, the support for it +is not installed by default. It is packaged as a separate install option, +available by clicking the **Customize** button on the **Installation Type** +step of the installer as described above. + +.. image:: mac_installer_09_custom_install_free_threaded.png + +If the box next to the **Free-threaded Python** package name is checked, +a separate :file:`PythonT.framework` will also be installed +alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. +This configuration allows a free-threaded Python |usemac_x_dot_y| build to co-exist +on your system with a traditional (GIL only) Python |usemac_x_dot_y| build with +minimal risk while installing or testing. This installation layout is itself +experimental and is subject to change in future releases. + +Known cautions and limitations: + +- The **UNIX command-line tools** package, which is selected by default, + will install links in :file:`/usr/local/bin` for |usemac_python_x_dot_y_t_literal|, + the free-threaded interpreter, and |usemac_python_x_dot_y_t_literal_config|, + a configuration utility which may be useful for package builders. + Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, + in most cases no changes to your ``PATH`` environment variables should + be needed to use |usemac_python_x_dot_y_t_literal|. + +- For this release, the **Shell profile updater** package and the + :file:`Update Shell Profile.command` in |usemac_applications_folder_version| + do not support the free-threaded package. + +- The free-threaded build and the traditional build have separate search + paths and separate :file:`site-packages` directories so, by default, + if you need a package available in both builds, it may need to be installed in both. + The free-threaded package will install a separate instance of :program:`pip` for use + with |usemac_python_x_dot_y_t_literal|. + + - To install a package using :command:`pip` without a :command:`venv`: + + |usemac_python_x_dot_y_t_literal| ``-m pip install `` + +- When working with multiple Python environments, it is usually safest and easiest + to :ref:`create and use virtual environments `. + This can avoid possible command name conflicts and confusion about which Python is in use: + + |usemac_python_x_dot_y_t_literal| ``-m venv `` + + then :command:`activate`. + +- To run a free-threaded version of IDLE: + + |usemac_python_x_dot_y_t_literal| ``-m idlelib`` + +- The interpreters in both builds respond to the same + :ref:`PYTHON environment variables ` + which may have unexpected results, for example, if you have ``PYTHONPATH`` + set in a shell profile. If necessary, there are + :ref:`command line options ` like ``-E`` + to ignore these environment variables. + +- The free-threaded build links to the third-party shared libraries, + such as ``OpenSSL`` and ``Tk``, installed in the traditional framework. + This means that both builds also share one set of trust certificates + as installed by the :command:`Install Certificates.command` script, + thus it only needs to be run once. + +- If you cannot depend on the link in ``/usr/local/bin`` pointing to the + ``python.org`` free-threaded |usemac_python_x_dot_y_t_literal| (for example, if you want + to install your own version there or some other distribution does), + you can explicitly set your shell ``PATH`` environment variable to + include the ``PythonT`` framework ``bin`` directory: + + .. code-block:: sh + + export PATH="/Library/Frameworks/PythonT.framework/Versions/3.13/bin":"$PATH" + + The traditional framework installation by default does something similar, + except for :file:`Python.framework`. Be aware that having both framework ``bin`` + directories in ``PATH`` can lead to confusion if there are duplicate names + like ``python3.13`` in both; which one is actually used depends on the order + they appear in ``PATH``. The ``which python3.x`` or ``which python3.xt`` + commands can show which path is being used. Using virtual environments + can help avoid such ambiguities. Another option might be to create + a shell :command:`alias` to the desired interpreter, like: + + .. code-block:: sh + + alias py3.13="/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13" + alias py3.13t="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t" + +Installing using the command line +--------------------------------- + +If you want to use automation to install the ``python.org`` installer package +(rather than by using the familiar macOS :program:`Installer` GUI app), +the macOS command line :command:`installer` utility lets you select non-default +options, too. If you are not familiar with :command:`installer`, it can be +somewhat cryptic (see :command:`man installer` for more information). +As an example, the following shell snippet shows one way to do it, +using the ``3.13.0b2`` release and selecting the free-threaded interpreter +option: + +.. code-block:: sh + + RELEASE="python-3.13.0b2-macos11.pkg" + + # download installer pkg + curl -O https://www.python.org/ftp/python/3.13.0/${RELEASE} + + # create installer choicechanges to customize the install: + # enable the PythonTFramework-3.13 package + # while accepting the other defaults (install all other packages) + cat > ./choicechanges.plist < + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.13 + + + + EOF + + sudo installer -pkg ./${RELEASE} -applyChoiceChangesXML ./choicechanges.plist -target / + + +You can then test that both installer builds are now available with something like: + +.. code-block:: console + + $ # test that the free-threaded interpreter was installed if the Unix Command Tools package was enabled + $ /usr/local/bin/python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # and the traditional interpreter + $ /usr/local/bin/python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # test that they are also available without the prefix if /usr/local/bin is on $PATH + $ python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + +.. note:: + + Current ``python.org`` installers only install to fixed locations like + :file:`/Library/Frameworks/`, :file:`/Applications`, and :file:`/usr/local/bin`. + You cannot use the :command:`installer` ``-domain`` option to install to + other locations. + .. _distributing-python-applications-on-the-mac: Distributing Python Applications -================================ +-------------------------------- A range of tools exist for converting your Python code into a standalone distributable application: @@ -180,12 +436,12 @@ distributable application: * :pypi:`py2app`: Supports creating macOS ``.app`` bundles from a Python project. -* `Briefcase `__: Part of the `BeeWare Project - `__; a cross-platform packaging tool that supports +* `Briefcase `_: Part of the `BeeWare Project + `_; a cross-platform packaging tool that supports creation of ``.app`` bundles on macOS, as well as managing signing and notarization. -* `PyInstaller `__: A cross-platform packaging tool that creates +* `PyInstaller `_: A cross-platform packaging tool that creates a single file or folder as a distributable artifact. App Store Compliance @@ -213,11 +469,6 @@ required if you are using the macOS App Store as a distribution channel. Other Resources =============== -The Pythonmac-SIG mailing list is an excellent support resource for Python users -and developers on the Mac: - -https://www.python.org/community/sigs/current/pythonmac-sig/ - -Another useful resource is the MacPython wiki: - -https://wiki.python.org/moin/MacPython +The `python.org Help page `_ has links to many useful resources. +The `Pythonmac-SIG mailing list `_ +is another support resource specifically for Python users and developers on the Mac. diff --git a/Doc/using/mac_installer_01_introduction.png b/Doc/using/mac_installer_01_introduction.png new file mode 100644 index 00000000000000..1999f3a3759093 Binary files /dev/null and b/Doc/using/mac_installer_01_introduction.png differ diff --git a/Doc/using/mac_installer_02_readme.png b/Doc/using/mac_installer_02_readme.png new file mode 100644 index 00000000000000..a36efaf7d50fd6 Binary files /dev/null and b/Doc/using/mac_installer_02_readme.png differ diff --git a/Doc/using/mac_installer_03_license.png b/Doc/using/mac_installer_03_license.png new file mode 100644 index 00000000000000..598c22a13d9e62 Binary files /dev/null and b/Doc/using/mac_installer_03_license.png differ diff --git a/Doc/using/mac_installer_04_installation_type.png b/Doc/using/mac_installer_04_installation_type.png new file mode 100644 index 00000000000000..9498fd06240a4e Binary files /dev/null and b/Doc/using/mac_installer_04_installation_type.png differ diff --git a/Doc/using/mac_installer_05_custom_install.png b/Doc/using/mac_installer_05_custom_install.png new file mode 100644 index 00000000000000..3a201d2f44655a Binary files /dev/null and b/Doc/using/mac_installer_05_custom_install.png differ diff --git a/Doc/using/mac_installer_06_summary.png b/Doc/using/mac_installer_06_summary.png new file mode 100644 index 00000000000000..1af6eee2c668cd Binary files /dev/null and b/Doc/using/mac_installer_06_summary.png differ diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png new file mode 100644 index 00000000000000..940219cad6f61c Binary files /dev/null and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/mac_installer_08_install_certificates.png b/Doc/using/mac_installer_08_install_certificates.png new file mode 100644 index 00000000000000..c125eeb18aa0c1 Binary files /dev/null and b/Doc/using/mac_installer_08_install_certificates.png differ diff --git a/Doc/using/mac_installer_09_custom_install_free_threaded.png b/Doc/using/mac_installer_09_custom_install_free_threaded.png new file mode 100644 index 00000000000000..0f69c55eddb228 Binary files /dev/null and b/Doc/using/mac_installer_09_custom_install_free_threaded.png differ diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc deleted file mode 100644 index 354eb1541ceac2..00000000000000 --- a/Doc/using/venv-create.inc +++ /dev/null @@ -1,121 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -command ``venv``:: - - python -m venv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation from which the command -was run (a common name for the target directory is ``.venv``). It also creates -a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy/symlink -of the Python binary/binaries (as appropriate for the platform or arguments -used at environment creation time). It also creates an (initially empty) -``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is -``Lib\site-packages``). If an existing directory is specified, it will be -re-used. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is - :ref:`deprecated in Python 3.6 `. - -.. highlight:: none - -On Windows, invoke the ``venv`` command as follows:: - - c:\>Python35\python -m venv c:\path\to\myenv - -Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for -your :ref:`Python installation `:: - - c:\>python -m venv c:\path\to\myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - [--without-scm-ignore-file] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if - it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been upgraded - in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-file - Skips adding the default SCM ignore file to the - environment directory (the default is a .gitignore - file). - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. - -.. versionchanged:: 3.13 - - ``--without-scm-ignore-file`` was added along with creating an ignore file - for ``git`` by default. - -.. versionchanged:: 3.12 - - ``setuptools`` is no longer a core venv dependency. - -.. versionchanged:: 3.9 - Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. - -.. note:: - While symlinks are supported on Windows, they are not recommended. Of - particular note is that double-clicking ``python.exe`` in File Explorer - will resolve the symlink eagerly and ignore the virtual environment. - -.. note:: - On Microsoft Windows, it may be required to enable the ``Activate.ps1`` - script by setting the execution policy for the user. You can do this by - issuing the following PowerShell command: - - PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - - See `About Execution Policies - `_ - for more information. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``venv``, in which case an identical virtual -environment will be created, according to the given options, at each provided -path. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 136236f51eb511..20d872d7639219 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -23,8 +23,9 @@ available for application-local distributions. As specified in :pep:`11`, a Python release only supports a Windows platform while Microsoft considers the platform under extended support. This means that -Python |version| supports Windows 8.1 and newer. If you require Windows 7 -support, please install Python 3.8. +Python |version| supports Windows 10 and newer. If you require Windows 7 +support, please install Python 3.8. If you require Windows 8.1 support, +please install Python 3.12. There are a number of different installers available for Windows, each with certain benefits and downsides. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 45817799b542bc..eae514a46810eb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -46,7 +46,7 @@ when researching a change. This article explains the new features in Python 3.13, compared to 3.12. -Python 3.13 will be released on October 1, 2024. +Python 3.13 will be released on October 7, 2024. For full details, see the :ref:`changelog `. .. seealso:: @@ -320,12 +320,9 @@ The free-threaded mode requires a different executable, usually called ``python3.13t`` or ``python3.13t.exe``. Pre-built binaries marked as *free-threaded* can be installed as part of the official :ref:`Windows ` -and :ref:`macOS ` installers, +and :ref:`macOS ` installers, or CPython can be built from source with the :option:`--disable-gil` option. -.. better macOS link pending - https://github.com/python/cpython/issues/109975#issuecomment-2286391179 - Free-threaded execution allows for full utilization of the available processing power by running threads in parallel on available CPU cores. While not all software will benefit from this automatically, programs @@ -824,6 +821,24 @@ copy (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* As a consequence of necessary internal refactoring, initialization of + internal metaclasses now happens in ``__init__`` rather + than in ``__new__``. This affects projects that subclass these internal + metaclasses to provide custom initialization. + Generally: + + - Custom logic that was done in ``__new__`` after calling ``super().__new__`` + should be moved to ``__init__``. + - To create a class, call the metaclass, not only the metaclass's + ``__new__`` method. + + See :gh:`124520` for discussion and links to changes in some affected + projects. + + dbm --- diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3d6084e6ecc19b..67d8d389b58082 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -91,7 +91,7 @@ annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VA format (which evaluates annotations to runtime values, similar to the behavior in earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format (which replaces undefined names with special markers), and the -:attr:`~annotationlib.Format.SOURCE` format (which returns annotations as strings). +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). This example shows how these formats behave: @@ -106,7 +106,7 @@ This example shows how these formats behave: NameError: name 'Undefined' is not defined >>> get_annotations(func, format=Format.FORWARDREF) {'arg': ForwardRef('Undefined')} - >>> get_annotations(func, format=Format.SOURCE) + >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} Implications for annotated code @@ -185,7 +185,7 @@ Other Language Changes ``python -O -c 'assert (__debug__ := 1)'`` now produces a :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) -* Added class methods :meth:`float.from_number` and :meth:`complex.from_number` +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` to convert a number to :class:`float` or :class:`complex` type correspondingly. They raise an error if the argument is a string. (Contributed by Serhiy Storchaka in :gh:`84978`.) @@ -202,11 +202,18 @@ New Modules Improved Modules ================ +argparse +-------- + +* The default value of the :ref:`program name ` for + :class:`argparse.ArgumentParser` now reflects the way the Python + interpreter was instructed to find the ``__main__`` module code. + (Contributed by Serhiy Storchaka and Alyssa Coghlan in :gh:`66436`.) ast --- -* Added :func:`ast.compare` for comparing two ASTs. +* Add :func:`ast.compare` for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) * Add support for :func:`copy.replace` for AST nodes. @@ -215,6 +222,9 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) +* The ``repr()`` output for AST nodes now includes more information. + (Contributed by Tomas R in :gh:`116022`.) + ctypes ------ @@ -233,9 +243,9 @@ ctypes dis --- -* Added support for rendering full source location information of +* Add support for rendering full source location information of :class:`instructions `, rather than only the line number. - This feature is added to the following interfaces via the ``show_positions`` + This feature is added to the following interfaces via the *show_positions* keyword argument: - :class:`dis.Bytecode`, @@ -243,22 +253,21 @@ dis - :func:`dis.disassemble`. This feature is also exposed via :option:`dis --show-positions`. - (Contributed by Bénédikt Tran in :gh:`123165`.) fractions --------- -Added support for converting any objects that have the -:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. -(Contributed by Serhiy Storchaka in :gh:`82017`.) +* Add support for converting any objects that have the + :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + (Contributed by Serhiy Storchaka in :gh:`82017`.) functools --------- -* Added support to :func:`functools.partial` and +* Add support to :func:`functools.partial` and :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels to reserve a place for positional arguments. (Contributed by Dominykas Grigonis in :gh:`119127`.) @@ -267,27 +276,27 @@ functools http ---- -Directory lists and error pages generated by the :mod:`http.server` -module allow the browser to apply its default dark mode. -(Contributed by Yorik Hansen in :gh:`123430`.) +* Directory lists and error pages generated by the :mod:`http.server` + module allow the browser to apply its default dark mode. + (Contributed by Yorik Hansen in :gh:`123430`.) json ---- -Add notes for JSON serialization errors that allow to identify the source -of the error. -(Contributed by Serhiy Storchaka in :gh:`122163`.) +* Add notes for JSON serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122163`.) -Enable :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. -See the :ref:`JSON command-line interface ` documentation. -(Contributed by Trey Hunner in :gh:`122873`.) +* Enable the :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. + See the :ref:`JSON command-line interface ` documentation. + (Contributed by Trey Hunner in :gh:`122873`.) operator -------- -* Two new functions ``operator.is_none`` and ``operator.is_not_none`` +* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` have been added, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. @@ -297,13 +306,13 @@ operator datetime -------- -Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. -(Contributed by Wannes Boeykens in :gh:`41431`.) +* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. + (Contributed by Wannes Boeykens in :gh:`41431`.) os -- -* Added the :data:`os.environ.refresh() ` method to update +* Add the :data:`os.environ.refresh() ` method to update :data:`os.environ` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) @@ -333,7 +342,7 @@ pdb :pdbcmd:`commands` are preserved across hard-coded breakpoints. (Contributed by Tian Gao in :gh:`121450`.) -* Added a new argument ``mode`` to :class:`pdb.Pdb`. Disabled ``restart`` +* Add a new argument *mode* to :class:`pdb.Pdb`. Disable the ``restart`` command when :mod:`pdb` is in ``inline`` mode. (Contributed by Tian Gao in :gh:`123757`.) @@ -341,7 +350,7 @@ pickle ------ * Set the default protocol version on the :mod:`pickle` module to 5. - For more details, please see :ref:`pickle protocols `. + For more details, see :ref:`pickle protocols `. * Add notes for pickle serialization errors that allow to identify the source of the error. @@ -379,12 +388,26 @@ asyncio Deprecated ========== +* :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`builtins`: Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) +* :mod:`multiprocessing` and :mod:`concurrent.futures`: + The default start method (see :ref:`multiprocessing-start-methods`) changed + away from *fork* to *forkserver* on platforms where it was not already + *spawn* (Windows & macOS). If you require the threading incompatible *fork* + start method you must explicitly specify it when using :mod:`multiprocessing` + or :mod:`concurrent.futures` APIs. + (Contributed by Gregory P. Smith in :gh:`84559`.) + * :mod:`os`: :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to @@ -429,7 +452,7 @@ ast user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom :class:`ast.NodeVisitor` subclasses will no longer be called when the - ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` method instead. Also, remove the following deprecated properties on :class:`ast.Constant`, @@ -580,18 +603,18 @@ New Features * Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` object: - * :c:func:`PyUnicodeWriter_Create`. - * :c:func:`PyUnicodeWriter_Discard`. - * :c:func:`PyUnicodeWriter_Finish`. - * :c:func:`PyUnicodeWriter_WriteChar`. - * :c:func:`PyUnicodeWriter_WriteUTF8`. - * :c:func:`PyUnicodeWriter_WriteUCS4`. - * :c:func:`PyUnicodeWriter_WriteWideChar`. - * :c:func:`PyUnicodeWriter_WriteStr`. - * :c:func:`PyUnicodeWriter_WriteRepr`. - * :c:func:`PyUnicodeWriter_WriteSubstring`. - * :c:func:`PyUnicodeWriter_Format`. - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteWideChar` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` (Contributed by Victor Stinner in :gh:`119182`.) @@ -603,11 +626,11 @@ New Features is backwards incompatible to any C-Extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will - normally result in crashes during the exectuion of the subsequent call to + normally result in crashes during the execution of the subsequent call to :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use an address sanitizer to identify any use-after-free coming from an interned string and deallocate it during module shutdown. - (Contribued by Eddie Elizondo in :gh:`113601`.) + (Contributed by Eddie Elizondo in :gh:`113601`.) * Add new functions to convert C ```` numbers from/to Python :class:`int`: @@ -683,12 +706,7 @@ Deprecated :c:macro:`!isfinite` available from :file:`math.h` since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) -* :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - -.. Add deprecations above alphabetically, not here at the end. +.. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst diff --git a/Include/Python.h b/Include/Python.h index 8fffa22df9da48..e1abdd16f031fb 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -55,6 +55,10 @@ # include // __readgsqword() #endif +#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) +# include // __readgsqword() +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 58d93fcfc1066b..03622698113ee7 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,6 +8,8 @@ extern "C" { #endif +/* Total tool ids available */ +#define _PY_MONITORING_TOOL_IDS 8 /* Count of all local monitoring events */ #define _PY_MONITORING_LOCAL_EVENTS 10 /* Count of all "real" monitoring events (not derived from other events) */ @@ -57,6 +59,8 @@ typedef struct { _Py_LocalMonitors active_monitors; /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; + /* The version of tools when they instrument the code */ + uintptr_t tool_versions[_PY_MONITORING_TOOL_IDS]; /* Information to support line events */ _PyCoLineInstrumentationData *lines; /* The tools that are to be notified for line events for the matching code unit */ diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index e2861c963266ea..b113c7fdcf6515 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -14,16 +14,11 @@ typedef struct { /* Number of items in the dictionary */ Py_ssize_t ma_used; - /* Dictionary version: globally unique, value change each time - the dictionary is modified */ -#ifdef Py_BUILD_CORE - /* Bits 0-7 are for dict watchers. + /* This is a private field for CPython's internal use. + * Bits 0-7 are for dict watchers. * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) - * The remaining bits (12-63) are the actual version tag. */ - uint64_t ma_version_tag; -#else - Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif + * The remaining bits are not currently used. */ + uint64_t _ma_watcher_tag; PyDictKeysObject *ma_keys; diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 82f8cc8a159c77..b239f7c557e016 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -71,10 +71,9 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); absolute value of a long. For example, this returns 1 for 1 and -1, 2 for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. v must not be NULL, and must be a normalized long. - (uint64_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. + Always successful. */ -PyAPI_FUNC(uint64_t) _PyLong_NumBits(PyObject *v); +PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v); /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3db3aa3eb77879..a9d1bce127e63d 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -108,7 +108,7 @@ backoff_counter_triggers(_Py_BackoffCounter counter) /* Initial JUMP_BACKWARD counter. * This determines when we create a trace for a loop. * Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ -#define JUMP_BACKWARD_INITIAL_VALUE 16 +#define JUMP_BACKWARD_INITIAL_VALUE 15 #define JUMP_BACKWARD_INITIAL_BACKOFF 4 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -122,7 +122,7 @@ initial_jump_backoff_counter(void) * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ -#define SIDE_EXIT_INITIAL_VALUE 64 +#define SIDE_EXIT_INITIAL_VALUE 63 #define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a97b53028c8f59..363845106e40dc 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -283,6 +283,7 @@ PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFra #define _PY_GC_SCHEDULED_BIT (1U << 4) #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) +#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 5e2d5c5ce9d868..4400be8b33dee7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -21,6 +21,17 @@ extern void _PyCodec_Fini(PyInterpreterState *interp); extern PyObject* _PyCodec_Lookup(const char *encoding); +/* + * Un-register the error handling callback function registered under + * the given 'name'. Only custom error handlers can be un-registered. + * + * - Return -1 and set an exception if 'name' refers to a built-in + * error handling name (e.g., 'strict'), or if an error occurred. + * - Return 0 if no custom error handler can be found for 'name'. + * - Return 1 if the custom error handler was successfully removed. + */ +extern int _PyCodec_UnregisterError(const char *name); + /* Text codec specific encoding and decoding API. Checks the encoding against a list of codecs which do not diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index f9a043b0208c8f..1920724c1d4f57 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -230,31 +230,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) -#ifdef Py_GIL_DISABLED - -#define THREAD_LOCAL_DICT_VERSION_COUNT 256 -#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT - -static inline uint64_t -dict_next_version(PyInterpreterState *interp) -{ - PyThreadState *tstate = PyThreadState_GET(); - uint64_t cur_progress = (tstate->dict_global_version & - (THREAD_LOCAL_DICT_VERSION_BATCH - 1)); - if (cur_progress == 0) { - uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version, - THREAD_LOCAL_DICT_VERSION_BATCH); - tstate->dict_global_version = next; - } - return tstate->dict_global_version += DICT_VERSION_INCREMENT; -} - -#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP) - -#else -#define DICT_NEXT_VERSION(INTERP) \ - ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) -#endif PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, @@ -263,7 +238,7 @@ _PyDict_SendEvent(int watcher_bits, PyObject *key, PyObject *value); -static inline uint64_t +static inline void _PyDict_NotifyEvent(PyInterpreterState *interp, PyDict_WatchEvent event, PyDictObject *mp, @@ -271,12 +246,11 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; + int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); } - return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 1a44755c7a01a3..11932b8d1e1ab6 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -12,10 +12,6 @@ extern "C" { #define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; uint32_t next_keys_version; PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 36366429e8db25..d7e584094f7839 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -35,7 +35,7 @@ extern "C" { #include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeid.h" // struct _Py_type_id_pool +#include "pycore_uniqueid.h" // struct _Py_unique_id_pool #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -221,7 +221,7 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state - struct _Py_type_id_pool type_ids; + struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; #endif @@ -261,7 +261,7 @@ struct _is { struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - + size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; @@ -272,6 +272,7 @@ struct _is { Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + uintptr_t monitoring_tool_versions[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 8822147b636dd4..196b4152280a35 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -79,11 +79,10 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) } // _PyLong_Frexp returns a double x and an exponent e such that the -// true value is approximately equal to x * 2**e. e is >= 0. x is +// true value is approximately equal to x * 2**e. x is // 0.0 if and only if the input is 0 (in which case, e and x are both -// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is -// possible if the number of bits doesn't fit into a Py_ssize_t, sets -// OverflowError and returns -1.0 for x, 0 for e. +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. +// Always successful. // // Export for 'math' shared extension PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e); @@ -105,10 +104,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 80b588815bc9cf..0af13b1bcda20b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,7 +14,7 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_typeid.h" // _PyType_IncrefSlow +#include "pycore_uniqueid.h" // _PyType_IncrefSlow #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) @@ -335,12 +335,12 @@ _Py_INCREF_TYPE(PyTypeObject *type) // Unsigned comparison so that `unique_id=-1`, which indicates that // per-thread refcounting has been disabled on this type, is handled by // the "else". - if ((size_t)ht->unique_id < (size_t)tstate->types.size) { + if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) { # ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); # endif _Py_INCREF_STAT_INC(); - tstate->types.refcounts[ht->unique_id]++; + tstate->refcounts.values[ht->unique_id]++; } else { // The slow path resizes the thread-local refcount array if necessary. @@ -368,12 +368,12 @@ _Py_DECREF_TYPE(PyTypeObject *type) // Unsigned comparison so that `unique_id=-1`, which indicates that // per-thread refcounting has been disabled on this type, is handled by // the "else". - if ((size_t)ht->unique_id < (size_t)tstate->types.size) { + if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) { # ifdef Py_REF_DEBUG _Py_DECREF_DecRefTotal(); # endif _Py_DECREF_STAT_INC(); - tstate->types.refcounts[ht->unique_id]--; + tstate->refcounts.values[ht->unique_id]--; } else { // Directly decref the type if the type id is not assigned or if diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 19e54bf122a8bb..f92c0a0cddf906 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -29,9 +29,10 @@ typedef struct { typedef struct { uint8_t opcode; uint8_t oparg; - uint16_t valid:1; - uint16_t linked:1; - uint16_t chain_depth:14; // Must be big engough for MAX_CHAIN_DEPTH - 1. + uint8_t valid:1; + uint8_t linked:1; + uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool warm; int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; @@ -123,11 +124,18 @@ PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #ifdef _Py_TIER2 PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); + #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) +# define _Py_Executors_InvalidateCold(A) ((void)0) + #endif +// Used as the threshold to trigger executor invalidation when +// trace_run_counter is greater than this value. +#define JIT_CLEANUP_THRESHOLD 100000 // This is the length of the trace we project initially. #define UOP_MAX_TRACE_LENGTH 800 diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index f681b644c9ad5d..a72ef4493b77ca 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -32,15 +32,15 @@ typedef struct _PyThreadStateImpl { struct _Py_freelists freelists; struct _brc_thread_state brc; struct { - // The thread-local refcounts for heap type objects - Py_ssize_t *refcounts; + // The per-thread refcounts + Py_ssize_t *values; // Size of the refcounts array. Py_ssize_t size; - // If set, don't use thread-local refcounts + // If set, don't use per-thread refcounts int is_finalized; - } types; + } refcounts; #endif #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) diff --git a/Include/internal/pycore_typeid.h b/Include/internal/pycore_typeid.h deleted file mode 100644 index e64d1447f6b51d..00000000000000 --- a/Include/internal/pycore_typeid.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef Py_INTERNAL_TYPEID_H -#define Py_INTERNAL_TYPEID_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#ifdef Py_GIL_DISABLED - -// This contains code for allocating unique ids to heap type objects -// and re-using those ids when the type is deallocated. -// -// The type ids are used to implement per-thread reference counts of -// heap type objects to avoid contention on the reference count fields -// of heap type objects. Static type objects are immortal, so contention -// is not an issue for those types. -// -// Type id of -1 is used to indicate a type doesn't use thread-local -// refcounting. This value is used when a type object is finalized by the GC -// and during interpreter shutdown to allow the type object to be -// deallocated promptly when the object's refcount reaches zero. -// -// Each entry implicitly represents a type id based on it's offset in the -// table. Non-allocated entries form a free-list via the 'next' pointer. -// Allocated entries store the corresponding PyTypeObject. -typedef union _Py_type_id_entry { - // Points to the next free type id, when part of the freelist - union _Py_type_id_entry *next; - - // Stores the type object when the id is assigned - PyHeapTypeObject *type; -} _Py_type_id_entry; - -struct _Py_type_id_pool { - PyMutex mutex; - - // combined table of types with allocated type ids and unallocated - // type ids. - _Py_type_id_entry *table; - - // Next entry to allocate inside 'table' or NULL - _Py_type_id_entry *freelist; - - // size of 'table' - Py_ssize_t size; -}; - -// Assigns the next id from the pool of type ids. -extern void _PyType_AssignId(PyHeapTypeObject *type); - -// Releases the allocated type id back to the pool. -extern void _PyType_ReleaseId(PyHeapTypeObject *type); - -// Merges the thread-local reference counts into the corresponding types. -extern void _PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate); - -// Like _PyType_MergeThreadLocalRefcounts, but also frees the thread-local -// array of refcounts. -extern void _PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate); - -// Frees the interpreter's pool of type ids. -extern void _PyType_FinalizeIdPool(PyInterpreterState *interp); - -// Increfs the type, resizing the thread-local refcount array if necessary. -PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); - -#endif /* Py_GIL_DISABLED */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_TYPEID_H */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index ca5a1e2adb4787..118bc98b35d5e3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -209,7 +209,6 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *); // Export for _testinternalcapi extension. PyAPI_FUNC(PyObject *) _PyType_GetSlotWrapperNames(void); diff --git a/Include/internal/pycore_uniqueid.h b/Include/internal/pycore_uniqueid.h new file mode 100644 index 00000000000000..8f3b4418408cf8 --- /dev/null +++ b/Include/internal/pycore_uniqueid.h @@ -0,0 +1,72 @@ +#ifndef Py_INTERNAL_UNIQUEID_H +#define Py_INTERNAL_UNIQUEID_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique ids to objects for per-thread +// reference counting. +// +// Per-thread reference counting is used along with deferred reference +// counting to avoid scaling bottlenecks due to reference count contention. +// +// An id of -1 is used to indicate that an object doesn't use per-thread +// refcounting. This value is used when the object is finalized by the GC +// and during interpreter shutdown to allow the object to be +// deallocated promptly when the object's refcount reaches zero. +// +// Each entry implicitly represents a unique id based on its offset in the +// table. Non-allocated entries form a free-list via the 'next' pointer. +// Allocated entries store the corresponding PyObject. +typedef union _Py_unique_id_entry { + // Points to the next free type id, when part of the freelist + union _Py_unique_id_entry *next; + + // Stores the object when the id is assigned + PyObject *obj; +} _Py_unique_id_entry; + +struct _Py_unique_id_pool { + PyMutex mutex; + + // combined table of object with allocated unique ids and unallocated ids. + _Py_unique_id_entry *table; + + // Next entry to allocate inside 'table' or NULL + _Py_unique_id_entry *freelist; + + // size of 'table' + Py_ssize_t size; +}; + +// Assigns the next id from the pool of ids. +extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj); + +// Releases the allocated id back to the pool. +extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id); + +// Merges the per-thread reference counts into the corresponding objects. +extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread +// array of refcounts. +extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Frees the interpreter's pool of type ids. +extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp); + +// Increfs the type, resizing the per-thread refcount array if necessary. +PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNIQUEID_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b950f760d74ac7..927dae88c1fa73 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -222,64 +222,65 @@ extern "C" { #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAKE_WARM 439 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 439 -#define _MONITOR_CALL 440 -#define _MONITOR_JUMP_BACKWARD 441 -#define _MONITOR_RESUME 442 +#define _MAYBE_EXPAND_METHOD 440 +#define _MONITOR_CALL 441 +#define _MONITOR_JUMP_BACKWARD 442 +#define _MONITOR_RESUME 443 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 443 -#define _POP_JUMP_IF_TRUE 444 +#define _POP_JUMP_IF_FALSE 444 +#define _POP_JUMP_IF_TRUE 445 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 445 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 446 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 446 +#define _PUSH_FRAME 447 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 447 -#define _PY_FRAME_KW 448 -#define _QUICKEN_RESUME 449 -#define _REPLACE_WITH_TRUE 450 +#define _PY_FRAME_GENERAL 448 +#define _PY_FRAME_KW 449 +#define _QUICKEN_RESUME 450 +#define _REPLACE_WITH_TRUE 451 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 451 -#define _SEND 452 -#define _SEND_GEN_FRAME 453 +#define _SAVE_RETURN_OFFSET 452 +#define _SEND 453 +#define _SEND_GEN_FRAME 454 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 454 -#define _STORE_ATTR 455 -#define _STORE_ATTR_INSTANCE_VALUE 456 -#define _STORE_ATTR_SLOT 457 -#define _STORE_ATTR_WITH_HINT 458 +#define _START_EXECUTOR 455 +#define _STORE_ATTR 456 +#define _STORE_ATTR_INSTANCE_VALUE 457 +#define _STORE_ATTR_SLOT 458 +#define _STORE_ATTR_WITH_HINT 459 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 459 -#define _STORE_FAST_0 460 -#define _STORE_FAST_1 461 -#define _STORE_FAST_2 462 -#define _STORE_FAST_3 463 -#define _STORE_FAST_4 464 -#define _STORE_FAST_5 465 -#define _STORE_FAST_6 466 -#define _STORE_FAST_7 467 +#define _STORE_FAST 460 +#define _STORE_FAST_0 461 +#define _STORE_FAST_1 462 +#define _STORE_FAST_2 463 +#define _STORE_FAST_3 464 +#define _STORE_FAST_4 465 +#define _STORE_FAST_5 466 +#define _STORE_FAST_6 467 +#define _STORE_FAST_7 468 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 468 -#define _STORE_SUBSCR 469 +#define _STORE_SLICE 469 +#define _STORE_SUBSCR 470 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 470 -#define _TO_BOOL 471 +#define _TIER2_RESUME_CHECK 471 +#define _TO_BOOL 472 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -289,14 +290,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 472 +#define _UNPACK_SEQUENCE 473 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 472 +#define MAX_UOP_ID 473 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4d0ab22e6aa8f3..07606135d7a356 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -274,6 +274,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, + [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, @@ -481,6 +482,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_WARM] = "_MAKE_WARM", [_MAP_ADD] = "_MAP_ADD", [_MATCH_CLASS] = "_MATCH_CLASS", [_MATCH_KEYS] = "_MATCH_KEYS", @@ -1062,6 +1064,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _START_EXECUTOR: return 0; + case _MAKE_WARM: + return 0; case _FATAL_ERROR: return 0; case _CHECK_VALIDITY_AND_SET_IP: diff --git a/Include/object.h b/Include/object.h index 7124f58f6bdb37..418f2196062df7 100644 --- a/Include/object.h +++ b/Include/object.h @@ -180,6 +180,12 @@ _Py_ThreadId(void) tid = __readfsdword(24); #elif defined(_MSC_VER) && defined(_M_ARM64) tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); #elif defined(__i386__) __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) diff --git a/Include/py_curses.h b/Include/py_curses.h index 79b1b01fcfa594..e11bfedb17d205 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -81,8 +81,6 @@ typedef struct { char *encoding; } PyCursesWindowObject; -#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) - #define PyCurses_CAPSULE_NAME "_curses._C_API" @@ -99,6 +97,8 @@ static void **PyCurses_API; #define PyCursesInitialised {if (! ((int (*)(void))PyCurses_API[2]) () ) return NULL;} #define PyCursesInitialisedColor {if (! ((int (*)(void))PyCurses_API[3]) () ) return NULL;} +#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) + #define import_curses() \ PyCurses_API = (void **)PyCapsule_Import(PyCurses_CAPSULE_NAME, 1); diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 358e2c070cd5fa..e0d20632516142 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -72,7 +72,7 @@ We currently also immortalize strings contained in code objects and similar, specifically in the compiler and in `marshal`. These are “close enough” to immortal: even in use cases like hot reloading or `eval`-ing user input, the number of distinct identifiers and string -constants expected to stay low. +constants is expected to stay low. ## Internal API diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 4139cbadf93e13..c2edf6c8856c21 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -485,10 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() - from annotationlib import value_to_source + from annotationlib import value_to_string return (f'collections.abc.Callable' - f'[[{", ".join([value_to_source(a) for a in self.__args__[:-1]])}], ' - f'{value_to_source(self.__args__[-1])}]') + f'[[{", ".join([value_to_string(a) for a in self.__args__[:-1]])}], ' + f'{value_to_string(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index f7a0095d795ac6..d457d2b5a338eb 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -371,15 +371,19 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self) -> INPUT_RECORD | None: + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): raise WinError(GetLastError()) - if read.value == 0: - return None - return rec def get_event(self, block: bool = True) -> Event | None: @@ -390,10 +394,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.pop() while True: - rec = self._read_input() + rec = self._read_input(block) if rec is None: - if block: - continue return None if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: @@ -464,8 +466,8 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" - while self._read_input() is not None: - pass + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -590,6 +592,14 @@ class INPUT_RECORD(Structure): ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] ReadConsoleInput.restype = BOOL + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) InHandle = GetStdHandle(STD_INPUT_HANDLE) else: @@ -602,5 +612,7 @@ def _win_only(*args, **kwargs): ScrollConsoleScreenBuffer = _win_only SetConsoleMode = _win_only ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only OutHandle = 0 InHandle = 0 diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index a027f4de3dfed6..d5166170c071c4 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -15,15 +15,15 @@ "call_evaluate_function", "get_annotate_function", "get_annotations", - "annotations_to_source", - "value_to_source", + "annotations_to_string", + "value_to_string", ] class Format(enum.IntEnum): VALUE = 1 FORWARDREF = 2 - SOURCE = 3 + STRING = 3 _Union = None @@ -291,9 +291,21 @@ def __convert_to_ast(self, other): return other.__ast_node__ elif isinstance(other, slice): return ast.Slice( - lower=self.__convert_to_ast(other.start) if other.start is not None else None, - upper=self.__convert_to_ast(other.stop) if other.stop is not None else None, - step=self.__convert_to_ast(other.step) if other.step is not None else None, + lower=( + self.__convert_to_ast(other.start) + if other.start is not None + else None + ), + upper=( + self.__convert_to_ast(other.stop) + if other.stop is not None + else None + ), + step=( + self.__convert_to_ast(other.step) + if other.step is not None + else None + ), ) else: return ast.Constant(value=other) @@ -469,7 +481,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): can be called with any of the format arguments in the Format enum, but compiler-generated __annotate__ functions only support the VALUE format. This function provides additional functionality to call __annotate__ - functions with the FORWARDREF and SOURCE formats. + functions with the FORWARDREF and STRING formats. *annotate* must be an __annotate__ function, which takes a single argument and returns a dict of annotations. @@ -487,8 +499,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): return annotate(format) except NotImplementedError: pass - if format == Format.SOURCE: - # SOURCE is implemented by calling the annotate function in a special + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special # environment where every name lookup results in an instance of _Stringifier. # _Stringifier supports every dunder operation and returns a new _Stringifier. # At the end, we get a dictionary that mostly contains _Stringifier objects (or @@ -524,9 +536,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): for key, val in annos.items() } elif format == Format.FORWARDREF: - # FORWARDREF is implemented similarly to SOURCE, but there are two changes, + # FORWARDREF is implemented similarly to STRING, but there are two changes, # at the beginning and the end of the process. - # First, while SOURCE uses an empty dictionary as the namespace, so that all + # First, while STRING uses an empty dictionary as the namespace, so that all # name lookups result in _Stringifier objects, FORWARDREF uses the globals # and builtins, so that defined names map to their real values. # Second, instead of returning strings, we want to return either real values @@ -673,11 +685,9 @@ def get_annotations( case Format.FORWARDREF: # For FORWARDREF, we use __annotations__ if it exists try: - ann = _get_dunder_annotations(obj) + return dict(_get_dunder_annotations(obj)) except NameError: pass - else: - return dict(ann) # But if __annotations__ threw a NameError, we try calling __annotate__ ann = _get_and_call_annotate(obj, format) @@ -688,14 +698,14 @@ def get_annotations( # __annotations__ threw NameError and there is no __annotate__. In that case, # we fall back to trying __annotations__ again. return dict(_get_dunder_annotations(obj)) - case Format.SOURCE: - # For SOURCE, we try to call __annotate__ + case Format.STRING: + # For STRING, we try to call __annotate__ ann = _get_and_call_annotate(obj, format) if ann is not None: return ann # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) - return annotations_to_source(ann) + return annotations_to_string(ann) case _: raise ValueError(f"Unsupported format {format!r}") @@ -764,10 +774,10 @@ def get_annotations( return return_value -def value_to_source(value): - """Convert a Python value to a format suitable for use with the SOURCE format. +def value_to_string(value): + """Convert a Python value to a format suitable for use with the STRING format. - This is inteded as a helper for tools that support the SOURCE format but do + This is inteded as a helper for tools that support the STRING format but do not have access to the code that originally produced the annotations. It uses repr() for most objects. @@ -783,10 +793,10 @@ def value_to_source(value): return repr(value) -def annotations_to_source(annotations): - """Convert an annotation dict containing values to approximately the SOURCE format.""" +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format.""" return { - n: t if isinstance(t, str) else value_to_source(t) + n: t if isinstance(t, str) else value_to_string(t) for n, t in annotations.items() } diff --git a/Lib/argparse.py b/Lib/argparse.py index 690b2a9db9481b..4b12c2f0c6f857 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1224,7 +1224,8 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): @@ -1696,6 +1697,28 @@ def add_mutually_exclusive_group(self, *args, **kwargs): return super().add_mutually_exclusive_group(*args, **kwargs) +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' + + class ArgumentParser(_AttributeHolder, _ActionsContainer): """Object for parsing command line strings into Python objects. @@ -1739,11 +1762,7 @@ def __init__(self, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1927,11 +1946,11 @@ def _parse_known_args(self, arg_strings, namespace): # otherwise, add the arg to the arg strings # and note the index if it was an option else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: + option_tuples = self._parse_optional(arg_string) + if option_tuples is None: pattern = 'A' else: - option_string_indices[i] = option_tuple + option_string_indices[i] = option_tuples pattern = 'O' arg_string_pattern_parts.append(pattern) @@ -1966,8 +1985,16 @@ def take_action(action, argument_strings, option_string=None): def consume_optional(start_index): # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, sep, explicit_arg = option_tuple + option_tuples = option_string_indices[start_index] + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, sep, explicit_arg in option_tuples]) + args = {'option': arg_string, 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + raise ArgumentError(None, msg % args) + + action, option_string, sep, explicit_arg = option_tuples[0] # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2253,7 +2280,7 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None, None + return [(action, arg_string, None, None)] # if it's just a single character, it was meant to be positional if len(arg_string) == 1: @@ -2263,25 +2290,14 @@ def _parse_optional(self, arg_string): option_string, sep, explicit_arg = arg_string.partition('=') if sep and option_string in self._option_string_actions: action = self._option_string_actions[option_string] - return action, option_string, sep, explicit_arg + return [(action, option_string, sep, explicit_arg)] # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, sep, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - raise ArgumentError(None, msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple + if option_tuples: + return option_tuples # if it was not found as an option, but it looks like a negative # number, it was meant to be positional @@ -2296,7 +2312,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None, None + return [(None, arg_string, None, None)] def _get_option_tuples(self, option_string): result = [] @@ -2319,7 +2335,9 @@ def _get_option_tuples(self, option_string): # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] @@ -2328,9 +2346,9 @@ def _get_option_tuples(self, option_string): action = self._option_string_actions[option_string] tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, None, None + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here @@ -2344,43 +2362,40 @@ def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs + # if this is an optional action, -- is not allowed + option = action.option_strings # the default (None) is assumed to be a single argument if nargs is None: - nargs_pattern = '(-*A-*)' + nargs_pattern = '([A])' if option else '(-*A-*)' # allow zero or one arguments elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' + nargs_pattern = '(A?)' if option else '(-*A?-*)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' + nargs_pattern = '(A*)' if option else '(-*[A-]*)' # allow one or more arguments elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' + nargs_pattern = '(A+)' if option else '(-*A[A-]*)' # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' + nargs_pattern = '([AO]*)' if option else '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' + nargs_pattern = '(A[AO]*)' if option else '(-*A[-AO]*)' # suppress action, like nargs=0 elif nargs == SUPPRESS: - nargs_pattern = '(-*-*)' + nargs_pattern = '()' if option else '(-*)' # all others should be integers else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') + nargs_pattern = '([AO]{%d})' % nargs if option else '((?:-*A){%d}-*)' % nargs # return the pattern return nargs_pattern @@ -2483,9 +2498,8 @@ def _get_values(self, action, arg_strings): value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2493,11 +2507,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2554,11 +2565,15 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + choices = action.choices + if choices is not None: + if isinstance(choices, str): + choices = iter(choices) + if value not in choices: + args = {'value': value, + 'choices': ', '.join(map(repr, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 5120140e061691..95c636f9e02866 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,6 +1,7 @@ import ast import asyncio import concurrent.futures +import contextvars import inspect import os import site @@ -22,6 +23,7 @@ def __init__(self, locals, loop): self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop + self.context = contextvars.copy_context() def runcode(self, code): global return_code @@ -55,12 +57,12 @@ def callback(): return try: - repl_future = self.loop.create_task(coro) + repl_future = self.loop.create_task(coro, context=self.context) futures._chain_future(repl_future, future) except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback) + loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index ffcc0174e1e245..000647f57dd9e3 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1144,7 +1144,7 @@ async def create_connection( (functools.partial(self._connect_sock, exceptions, addrinfo, laddr_infos) for addrinfo in infos), - happy_eyeballs_delay) + happy_eyeballs_delay, loop=self) if sock is None: exceptions = [exc for sub in exceptions for exc in sub] diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 4458d01dece0e6..c3a7441a7b091d 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -4,14 +4,13 @@ import contextlib +from . import events +from . import exceptions as exceptions_mod from . import locks from . import tasks -from . import taskgroups -class _Done(Exception): - pass -async def staggered_race(coro_fns, delay): +async def staggered_race(coro_fns, delay, *, loop=None): """Run coroutines with staggered start times and take the first to finish. This method takes an iterable of coroutine functions. The first one is @@ -43,6 +42,8 @@ async def staggered_race(coro_fns, delay): delay: amount of time, in seconds, between starting coroutines. If ``None``, the coroutines will run sequentially. + loop: the event loop to use. + Returns: tuple *(winner_result, winner_index, exceptions)* where @@ -61,11 +62,36 @@ async def staggered_race(coro_fns, delay): """ # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. + loop = loop or events.get_running_loop() + enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None exceptions = [] + running_tasks = [] + + async def run_one_coro(previous_failed) -> None: + # Wait for the previous task to finish, or for delay seconds + if previous_failed is not None: + with contextlib.suppress(exceptions_mod.TimeoutError): + # Use asyncio.wait_for() instead of asyncio.wait() here, so + # that if we get cancelled at this point, Event.wait() is also + # cancelled, otherwise there will be a "Task destroyed but it is + # pending" later. + await tasks.wait_for(previous_failed.wait(), delay) + # Get the next coroutine to run + try: + this_index, coro_fn = next(enum_coro_fns) + except StopIteration: + return + # Start task that will run the next coroutine + this_failed = locks.Event() + next_task = loop.create_task(run_one_coro(this_failed)) + running_tasks.append(next_task) + assert len(running_tasks) == this_index + 2 + # Prepare place to put this coroutine's exceptions if not won + exceptions.append(None) + assert len(exceptions) == this_index + 1 - async def run_one_coro(this_index, coro_fn, this_failed): try: result = await coro_fn() except (SystemExit, KeyboardInterrupt): @@ -79,17 +105,34 @@ async def run_one_coro(this_index, coro_fn, this_failed): assert winner_index is None winner_index = this_index winner_result = result - raise _Done - + # Cancel all other tasks. We take care to not cancel the current + # task as well. If we do so, then since there is no `await` after + # here and CancelledError are usually thrown at one, we will + # encounter a curious corner case where the current task will end + # up as done() == True, cancelled() == False, exception() == + # asyncio.CancelledError. This behavior is specified in + # https://bugs.python.org/issue30048 + for i, t in enumerate(running_tasks): + if i != this_index: + t.cancel() + + first_task = loop.create_task(run_one_coro(None)) + running_tasks.append(first_task) try: - async with taskgroups.TaskGroup() as tg: - for this_index, coro_fn in enumerate(coro_fns): - this_failed = locks.Event() - exceptions.append(None) - tg.create_task(run_one_coro(this_index, coro_fn, this_failed)) - with contextlib.suppress(TimeoutError): - await tasks.wait_for(this_failed.wait(), delay) - except* _Done: - pass - - return winner_result, winner_index, exceptions + # Wait for a growing list of tasks to all finish: poor man's version of + # curio's TaskGroup or trio's nursery + done_count = 0 + while done_count != len(running_tasks): + done, _ = await tasks.wait(running_tasks) + done_count = len(done) + # If run_one_coro raises an unhandled exception, it's probably a + # programming error, and I want to see it. + if __debug__: + for d in done: + if d.done() and not d.cancelled() and d.exception(): + raise d.exception() + return winner_result, winner_index, exceptions + finally: + # Make sure no tasks are left running if we leave this function + for t in running_tasks: + t.cancel() diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index bff76291634604..034ba377a0dbec 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ -from _collections_abc import * -from _collections_abc import __all__ # noqa: F401 -from _collections_abc import _CallableGenericAlias # noqa: F401 +import _collections_abc +import sys +sys.modules[__name__] = _collections_abc diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index f5cb97edaf72cd..7a24f8a9e5ccee 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -283,11 +283,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -300,6 +301,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None @recursive_repr() @@ -315,6 +317,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -382,7 +385,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -394,7 +397,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -402,7 +405,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -1174,7 +1177,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1239,7 +1242,32 @@ def _update_func_cell_for__class__(f, oldcls, newcls): return False -def _add_slots(cls, is_frozen, weakref_slot): +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots.update({slot: doc}) + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # Need to create a new class, since we can't set __slots__ after a # class has been created, and the @dataclass decorator is called # after the class is created. @@ -1255,17 +1283,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: @@ -1530,7 +1550,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None): + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1610,8 +1630,8 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + # Apply the normal provided decorator. + return decorator(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, match_args=match_args, kw_only=kw_only, slots=slots, weakref_slot=weakref_slot) diff --git a/Lib/decimal.py b/Lib/decimal.py index f8c548eb1c6ecf..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -103,6 +103,7 @@ from _decimal import __version__ # noqa: F401 from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __version__ # noqa: F401 - from _pydecimal import __libmpdec_version__ # noqa: F401 + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/functools.py b/Lib/functools.py index 83b8895794e7c0..9d53d3601559b2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -16,15 +16,12 @@ from abc import get_cache_token from collections import namedtuple -# import types, weakref # Deferred to single_dispatch() +# import weakref # Deferred to single_dispatch() from operator import itemgetter from reprlib import recursive_repr -from types import MethodType +from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock -# Avoid importing types, so we can speedup import time -GenericAlias = type(list[int]) - ################################################################################ ### update_wrapper() and wraps() decorator ################################################################################ @@ -900,7 +897,7 @@ def singledispatch(func): # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. - import types, weakref + import weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() @@ -931,7 +928,7 @@ def dispatch(cls): def _is_union_type(cls): from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} + return get_origin(cls) in {Union, UnionType} def _is_valid_dispatch_type(cls): if isinstance(cls, type): @@ -1008,7 +1005,7 @@ def wrapper(*args, **kw): registry[object] = func wrapper.register = register wrapper.dispatch = dispatch - wrapper.registry = types.MappingProxyType(registry) + wrapper.registry = MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b25300fcb2509..17314564f35397 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -970,10 +970,12 @@ def findsource(object): if isclass(object): try: - firstlineno = vars(object)['__firstlineno__'] + lnum = vars(object)['__firstlineno__'] - 1 except (TypeError, KeyError): raise OSError('source code not available') - return lines, firstlineno - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index ddcc7e7900999e..d0a3ad00e53ad8 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -259,13 +259,12 @@ def get_start_method(self, allow_none=False): def get_all_start_methods(self): """Returns a list of the supported start methods, default first.""" - if sys.platform == 'win32': - return ['spawn'] - else: - methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] - if reduction.HAVE_SEND_HANDLE: - methods.append('forkserver') - return methods + default = self._default_context.get_start_method() + start_method_names = [default] + start_method_names.extend( + name for name in _concrete_contexts if name != default + ) + return start_method_names # @@ -320,14 +319,15 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - if sys.platform == 'darwin': - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - _default_context = DefaultContext(_concrete_contexts['spawn']) + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + _default_context = DefaultContext(_concrete_contexts['forkserver']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts['spawn']) -else: +else: # Windows class SpawnProcess(process.BaseProcess): _start_method = 'spawn' diff --git a/Lib/pdb.py b/Lib/pdb.py index 443160eaaae887..aea6fb70ae3106 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -350,10 +350,6 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, pass self.commands = {} # associates a command list to breakpoint numbers - self.commands_doprompt = {} # for each bp num, tells if the prompt - # must be disp. after execing the cmd list - self.commands_silent = {} # for each bp num, tells if the stack trace - # must be disp. after execing the cmd list self.commands_defining = False # True while in the process of defining # a command list self.commands_bnum = None # The breakpoint number for which we are @@ -403,13 +399,6 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary used to be updated from the actual frame - # locals whenever the .f_locals accessor was called, so it was - # cached here to ensure that modifications were not overwritten. While - # the caching is no longer required now that f_locals is a direct proxy - # on optimized frames, it's also harmless, so the code structure has - # been left unchanged. - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) if self._chained_exceptions: @@ -444,8 +433,8 @@ def user_line(self, frame): or frame.f_lineno <= 0): return self._wait_for_mainpyfile = False - if self.bp_commands(frame): - self.interaction(frame, None) + self.bp_commands(frame) + self.interaction(frame, None) user_opcode = user_line @@ -460,18 +449,9 @@ def bp_commands(self, frame): self.currentbp in self.commands: currentbp = self.currentbp self.currentbp = 0 - lastcmd_back = self.lastcmd - self.setup(frame, None) for line in self.commands[currentbp]: - self.onecmd(line) - self.lastcmd = lastcmd_back - if not self.commands_silent[currentbp]: - self.print_stack_entry(self.stack[self.curindex]) - if self.commands_doprompt[currentbp]: - self._cmdloop() - self.forget() - return - return 1 + self.cmdqueue.append(line) + self.cmdqueue.append(f'_pdbcmd_restore_lastcmd {self.lastcmd}') def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" @@ -732,7 +712,7 @@ def _exec_in_closure(self, source, globals, locals): def default(self, line): if line[:1] == '!': line = line[1:].strip() - locals = self.curframe_locals + locals = self.curframe.f_locals globals = self.curframe.f_globals try: buffer = line @@ -870,15 +850,15 @@ def handle_command_def(self, line): cmd, arg, line = self.parseline(line) if not cmd: return False - if cmd == 'silent': - self.commands_silent[self.commands_bnum] = True - return False # continue to handle other cmd def in the cmd list - elif cmd == 'end': + if cmd == 'end': return True # end of cmd list elif cmd == 'EOF': print('') return True # end of cmd list cmdlist = self.commands[self.commands_bnum] + if cmd == 'silent': + cmdlist.append('_pdbcmd_silence_frame_status') + return False # continue to handle other cmd def in the cmd list if arg: cmdlist.append(cmd+' '+arg) else: @@ -890,7 +870,6 @@ def handle_command_def(self, line): func = self.default # one of the resuming commands if func.__name__ in self.commands_resuming: - self.commands_doprompt[self.commands_bnum] = False return True return False @@ -960,7 +939,7 @@ def _complete_expression(self, text, line, begidx, endidx): # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} if text.startswith("$"): # Complete convenience variables conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) @@ -991,7 +970,7 @@ def completedefault(self, text, line, begidx, endidx): # Use rlcompleter to do the completion state = 0 matches = [] - completer = Completer(self.curframe.f_globals | self.curframe_locals) + completer = Completer(self.curframe.f_globals | self.curframe.f_locals) while (match := completer.complete(text, state)) is not None: matches.append(match) state += 1 @@ -1003,6 +982,13 @@ def _pdbcmd_print_frame_status(self, arg): self.print_stack_trace(0) self._show_display() + def _pdbcmd_silence_frame_status(self, arg): + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() + + def _pdbcmd_restore_lastcmd(self, arg): + self.lastcmd = arg + # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop @@ -1061,14 +1047,10 @@ def do_commands(self, arg): self.commands_bnum = bnum # Save old definitions for the case of a keyboard interrupt. if bnum in self.commands: - old_command_defs = (self.commands[bnum], - self.commands_doprompt[bnum], - self.commands_silent[bnum]) + old_commands = self.commands[bnum] else: - old_command_defs = None + old_commands = None self.commands[bnum] = [] - self.commands_doprompt[bnum] = True - self.commands_silent[bnum] = False prompt_back = self.prompt self.prompt = '(com) ' @@ -1077,14 +1059,10 @@ def do_commands(self, arg): self.cmdloop() except KeyboardInterrupt: # Restore old definitions. - if old_command_defs: - self.commands[bnum] = old_command_defs[0] - self.commands_doprompt[bnum] = old_command_defs[1] - self.commands_silent[bnum] = old_command_defs[2] + if old_commands: + self.commands[bnum] = old_commands else: del self.commands[bnum] - del self.commands_doprompt[bnum] - del self.commands_silent[bnum] self.error('command definition aborted, old commands restored') finally: self.commands_defining = False @@ -1153,7 +1131,7 @@ def do_break(self, arg, temporary = 0): try: func = eval(arg, self.curframe.f_globals, - self.curframe_locals) + self.curframe.f_locals) except: func = arg try: @@ -1458,7 +1436,6 @@ def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None @@ -1704,7 +1681,7 @@ def do_debug(self, arg): """ sys.settrace(None) globals = self.curframe.f_globals - locals = self.curframe_locals + locals = self.curframe.f_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") @@ -1749,7 +1726,7 @@ def do_args(self, arg): self._print_invalid_arg(arg) return co = self.curframe.f_code - dict = self.curframe_locals + dict = self.curframe.f_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 @@ -1769,15 +1746,15 @@ def do_retval(self, arg): if arg: self._print_invalid_arg(arg) return - if '__return__' in self.curframe_locals: - self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) + if '__return__' in self.curframe.f_locals: + self.message(self._safe_repr(self.curframe.f_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) except: self._error_exc() raise @@ -1785,7 +1762,7 @@ def _getval(self, arg): def _getval_except(self, arg, frame=None): try: if frame is None: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except BaseException as exc: @@ -2029,7 +2006,7 @@ def do_interact(self, arg): Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} console = _PdbInteractiveConsole(ns, message=self.message) console.interact(banner="*pdb interact start*", exitmsg="*exit from pdb interact command*") @@ -2101,7 +2078,7 @@ def complete_unalias(self, text, line, begidx, endidx): # List of all the commands making the program resume execution. commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return', - 'do_quit', 'do_jump'] + 'do_until', 'do_quit', 'do_jump'] # Print a traceback starting at the top stack frame. # The most recently entered frame is printed last; diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d376592d69d40d..eec7b0770f56ca 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1870,6 +1870,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1963,6 +1964,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 4643df80e44aaf..97bb4eb52f4386 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -416,6 +416,34 @@ 'some expressions (like un-parenthesized tuple expressions) ' 'caused a\n' 'syntax error.\n', + 'assignment-expressions': 'Assignment expressions\n' + '**********************\n' + '\n' + 'An assignment expression (sometimes also called a “named expression”' + '\nor “walrus”) assigns an expression to an identifier, while also\n' + 'returning the value of the expression.\n' + '\n' + 'One common use case is when handling matched regular expressions:\n' + '\n' + ' if matching := pattern.search(data):\n' + ' do_something(matching)\n' + '\n' + 'Or, when processing a file stream in chunks:\n' + '\n' + ' while chunk := file.read(9000):\n' + ' process(chunk)\n' + '\n' + 'Assignment expressions must be surrounded by parentheses when used as\n' + 'expression statements and when used as sub-expressions in slicing,\n' + 'conditional, lambda, keyword-argument, and comprehension-if\n' + 'expressions and in assert, with, and assignment statements. In all\n' + 'other places where they can be used, parentheses are not required,\n' + 'including in if and while statements.\n' + '\n' + 'Added in version 3.8.\n' + 'See also:\n' + '\n' + ' **PEP 572** - Assignment Expressions\n', 'async': 'Coroutines\n' '**********\n' '\n' diff --git a/Lib/statistics.py b/Lib/statistics.py index d3dd0d530c31cf..f193fcdc241aa9 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -870,9 +870,12 @@ def f_inv(y): return f_inv def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: + if p < 0.499: x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) return x * sign @@ -886,8 +889,11 @@ def quartic_kernel(): return pdf, cdf, invcdf, support def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) return x * sign @register('triweight') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4b3a0645cfc84a..a059a6b8340448 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5553,15 +5553,29 @@ def test_set_get(self): multiprocessing.set_start_method(old_method, force=True) self.assertGreaterEqual(count, 1) - def test_get_all(self): + def test_get_all_start_methods(self): methods = multiprocessing.get_all_start_methods() + self.assertIn('spawn', methods) if sys.platform == 'win32': self.assertEqual(methods, ['spawn']) + elif sys.platform == 'darwin': + self.assertEqual(methods[0], 'spawn') # The default is first. + # Whether these work or not, they remain available on macOS. + self.assertIn('fork', methods) + self.assertIn('forkserver', methods) else: - self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['spawn', 'fork'] or - methods == ['fork', 'spawn', 'forkserver'] or - methods == ['spawn', 'fork', 'forkserver']) + # POSIX + self.assertIn('fork', methods) + if other_methods := set(methods) - {'fork', 'spawn'}: + # If there are more than those two, forkserver must be one. + self.assertEqual({'forkserver'}, other_methods) + # The default is the first method in the list. + self.assertIn(methods[0], {'forkserver', 'spawn'}, + msg='3.14+ default must not be fork') + if methods[0] == 'spawn': + # Confirm that the current default selection logic prefers + # forkserver vs spawn when available. + self.assertNotIn('forkserver', methods) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 7dcaf085a7ca91..d6be4ad049d14a 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -300,29 +300,78 @@ def get_build_info(): config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' cflags = sysconfig.get_config_var('PY_CFLAGS') or '' - cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' build = [] # --disable-gil if sysconfig.get_config_var('Py_GIL_DISABLED'): - build.append("free_threading") + if not sys.flags.ignore_environment: + PYTHON_GIL = os.environ.get('PYTHON_GIL', None) + if PYTHON_GIL: + PYTHON_GIL = (PYTHON_GIL == '1') + else: + PYTHON_GIL = None + + free_threading = "free_threading" + if PYTHON_GIL is not None: + free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" + build.append(free_threading) if hasattr(sys, 'gettotalrefcount'): # --with-pydebug build.append('debug') - if '-DNDEBUG' in (cflags + cflags_nodist): + if '-DNDEBUG' in cflags: build.append('without_assert') else: build.append('release') if '--with-assertions' in config_args: build.append('with_assert') - elif '-DNDEBUG' not in (cflags + cflags_nodist): + elif '-DNDEBUG' not in cflags: build.append('with_assert') + # --enable-experimental-jit + tier2 = re.search('-D_Py_TIER2=([0-9]+)', cflags) + if tier2: + tier2 = int(tier2.group(1)) + + if not sys.flags.ignore_environment: + PYTHON_JIT = os.environ.get('PYTHON_JIT', None) + if PYTHON_JIT: + PYTHON_JIT = (PYTHON_JIT != '0') + else: + PYTHON_JIT = None + + if tier2 == 1: # =yes + if PYTHON_JIT == False: + jit = 'JIT=off' + else: + jit = 'JIT' + elif tier2 == 3: # =yes-off + if PYTHON_JIT: + jit = 'JIT' + else: + jit = 'JIT=off' + elif tier2 == 4: # =interpreter + if PYTHON_JIT == False: + jit = 'JIT-interpreter=off' + else: + jit = 'JIT-interpreter' + elif tier2 == 6: # =interpreter-off (Secret option!) + if PYTHON_JIT: + jit = 'JIT-interpreter' + else: + jit = 'JIT-interpreter=off' + elif '-D_Py_JIT' in cflags: + jit = 'JIT' + else: + jit = None + if jit: + build.append(jit) + # --enable-framework=name framework = sysconfig.get_config_var('PYTHONFRAMEWORK') if framework: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 628529b8664c77..99cb10fc7b5f7b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2209,7 +2209,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index edcd2b9a35bbd9..2b91bdcf9cd859 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -58,8 +58,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f493a92e0ddce8..14cd50bd30502c 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -849,7 +849,6 @@ def test_execution_namespace_is_main(self): ns.pop('__loader__') self.assertEqual(ns, { '__name__': '__main__', - '__annotations__': {}, '__doc__': None, '__package__': None, '__spec__': None, diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index dc1106aee1e2f1..eedf2506a14912 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -12,8 +12,8 @@ ForwardRef, get_annotations, get_annotate_function, - annotations_to_source, - value_to_source, + annotations_to_string, + value_to_string, ) from typing import Unpack @@ -39,14 +39,14 @@ def __repr__(self): class TestFormat(unittest.TestCase): def test_enum(self): - self.assertEqual(annotationlib.Format.VALUE.value, 1) - self.assertEqual(annotationlib.Format.VALUE, 1) + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) - self.assertEqual(annotationlib.Format.FORWARDREF.value, 2) - self.assertEqual(annotationlib.Format.FORWARDREF, 2) + self.assertEqual(Format.FORWARDREF.value, 2) + self.assertEqual(Format.FORWARDREF, 2) - self.assertEqual(annotationlib.Format.SOURCE.value, 3) - self.assertEqual(annotationlib.Format.SOURCE, 3) + self.assertEqual(Format.STRING.value, 3) + self.assertEqual(Format.STRING, 3) class TestForwardRefFormat(unittest.TestCase): @@ -54,9 +54,7 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) fwdref = anno["arg"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) self.assertEqual(fwdref.__forward_arg__, "x") @@ -66,16 +64,14 @@ def inner(arg: x): x = 1 self.assertEqual(fwdref.evaluate(), x) - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) self.assertIs(anno["x"], int) fwdref = anno["y"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) @@ -92,14 +88,14 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations(inner, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(inner, format=Format.STRING) self.assertEqual(anno, {"arg": "x"}) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) def test_expressions(self): @@ -133,7 +129,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -184,7 +180,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -218,7 +214,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -241,13 +237,13 @@ def f(fstring: f"{a}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) def f(fstring_format: f"{a:02d}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) class TestForwardRefClass(unittest.TestCase): @@ -276,7 +272,7 @@ class Gen[T]: with self.assertRaises(NameError): ForwardRef("T").evaluate(owner=int) - T, = Gen.__type_params__ + (T,) = Gen.__type_params__ self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) @@ -294,8 +290,7 @@ class Gen[T]: def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) self.assertIs( - ForwardRef("Counter", module="collections").evaluate(), - collections.Counter + ForwardRef("Counter", module="collections").evaluate(), collections.Counter ) self.assertEqual( ForwardRef("Counter[int]", module="collections").evaluate(), @@ -383,22 +378,20 @@ class C1(metaclass=NoDict): self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(C1, format=Format.FORWARDREF), {"a": int}, ) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(C1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) self.assertEqual( - annotationlib.get_annotations( - NoDict, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(NoDict, format=Format.FORWARDREF), {"b": str}, ) self.assertEqual( - annotationlib.get_annotations(NoDict, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(NoDict, format=Format.STRING), {"b": "str"}, ) @@ -410,20 +403,20 @@ def f2(a: undefined): pass self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.VALUE), + annotationlib.get_annotations(f1, format=Format.VALUE), {"a": int}, ) self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) fwd = annotationlib.ForwardRef("undefined") self.assertEqual( - annotationlib.get_annotations(f2, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(f2, format=Format.FORWARDREF), {"a": fwd}, ) self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(f1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) @@ -446,30 +439,26 @@ def foo(): pass with self.assertRaises(ValueError): - annotationlib.get_annotations( - foo, format=annotationlib.Format.FORWARDREF, eval_str=True - ) - annotationlib.get_annotations( - foo, format=annotationlib.Format.SOURCE, eval_str=True - ) + annotationlib.get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + annotationlib.get_annotations(foo, format=Format.STRING, eval_str=True) def test_stock_annotations(self): def foo(a: int, b: str): pass - for format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + for format in (Format.VALUE, Format.FORWARDREF): with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), {"a": int, "b": str}, ) self.assertEqual( - annotationlib.get_annotations(foo, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(foo, format=Format.STRING), {"a": "int", "b": "str"}, ) foo.__annotations__ = {"a": "foo", "b": "str"} - for format in annotationlib.Format: + for format in Format: with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), @@ -491,10 +480,10 @@ def test_stock_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -529,7 +518,7 @@ def test_stock_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -563,48 +552,36 @@ def test_stock_annotations_in_module(self): ) self.assertEqual( - annotationlib.get_annotations(isa, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(isa, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.MyClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.MyClass, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function2, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function2, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function3, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function3, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - annotationlib, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(annotationlib, format=Format.STRING), {}, ) self.assertEqual( - annotationlib.get_annotations( - isa.UnannotatedClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.UnannotatedClass, format=Format.STRING), {}, ) self.assertEqual( annotationlib.get_annotations( - isa.unannotated_function, format=annotationlib.Format.SOURCE + isa.unannotated_function, format=Format.STRING ), {}, ) @@ -620,13 +597,11 @@ def test_stock_annotations_on_wrapper(self): {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations( - wrapped, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(wrapped, format=Format.FORWARDREF), {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations(wrapped, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(wrapped, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( @@ -643,12 +618,12 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.SOURCE}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, - {"format": annotationlib.Format.SOURCE, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -681,7 +656,7 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -767,9 +742,9 @@ def f(x: int): annotationlib.get_annotations(f, format=Format.FORWARDREF), {"x": str}, ) - # ... but not in SOURCE which always uses __annotate__ + # ... but not in STRING which always uses __annotate__ self.assertEqual( - annotationlib.get_annotations(f, format=Format.SOURCE), + annotationlib.get_annotations(f, format=Format.STRING), {"x": "int"}, ) @@ -804,7 +779,7 @@ def __annotations__(self): ) self.assertEqual( - annotationlib.get_annotations(ha, format=Format.SOURCE), {"x": "int"} + annotationlib.get_annotations(ha, format=Format.STRING), {"x": "int"} ) def test_raising_annotations_on_custom_object(self): @@ -844,7 +819,7 @@ def __annotate__(self): annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} ) self.assertEqual( - annotationlib.get_annotations(hb, format=Format.SOURCE), {"x": str} + annotationlib.get_annotations(hb, format=Format.STRING), {"x": str} ) def test_pep695_generic_class_with_future_annotations(self): @@ -974,15 +949,13 @@ def evaluate(format, exc=NotImplementedError): return undefined with self.assertRaises(NameError): - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.VALUE) + annotationlib.call_evaluate_function(evaluate, Format.VALUE) self.assertEqual( - annotationlib.call_evaluate_function( - evaluate, annotationlib.Format.FORWARDREF - ), + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), annotationlib.ForwardRef("undefined"), ) self.assertEqual( - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.SOURCE), + annotationlib.call_evaluate_function(evaluate, Format.STRING), "undefined", ) @@ -1093,25 +1066,25 @@ class C: class TestToSource(unittest.TestCase): - def test_value_to_source(self): - self.assertEqual(value_to_source(int), "int") - self.assertEqual(value_to_source(MyClass), "test.test_annotationlib.MyClass") - self.assertEqual(value_to_source(len), "len") - self.assertEqual(value_to_source(value_to_source), "value_to_source") - self.assertEqual(value_to_source(times_three), "times_three") - self.assertEqual(value_to_source(...), "...") - self.assertEqual(value_to_source(None), "None") - self.assertEqual(value_to_source(1), "1") - self.assertEqual(value_to_source("1"), "'1'") - self.assertEqual(value_to_source(Format.VALUE), repr(Format.VALUE)) - self.assertEqual(value_to_source(MyClass()), "my repr") - - def test_annotations_to_source(self): - self.assertEqual(annotations_to_source({}), {}) - self.assertEqual(annotations_to_source({"x": int}), {"x": "int"}) - self.assertEqual(annotations_to_source({"x": "int"}), {"x": "int"}) + def test_value_to_string(self): + self.assertEqual(value_to_string(int), "int") + self.assertEqual(value_to_string(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_string(len), "len") + self.assertEqual(value_to_string(value_to_string), "value_to_string") + self.assertEqual(value_to_string(times_three), "times_three") + self.assertEqual(value_to_string(...), "...") + self.assertEqual(value_to_string(None), "None") + self.assertEqual(value_to_string(1), "1") + self.assertEqual(value_to_string("1"), "'1'") + self.assertEqual(value_to_string(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_string(MyClass()), "my repr") + + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) self.assertEqual( - annotations_to_source({"x": int, "y": str}), {"x": "int", "y": "str"} + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} ) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ef05a6fefcffcc..057379cec91ba9 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6,6 +6,7 @@ import io import operator import os +import py_compile import shutil import stat import sys @@ -15,10 +16,16 @@ import argparse import warnings -from test.support import os_helper, captured_stderr +from test.support import captured_stderr +from test.support import import_helper +from test.support import os_helper +from test.support import script_helper from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -380,15 +387,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -621,9 +635,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -679,7 +693,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -875,7 +889,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] @@ -914,6 +930,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" @@ -1001,8 +1034,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1035,8 +1068,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1049,9 +1082,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1570,18 +1603,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] @@ -2238,14 +2277,14 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser @@ -2312,6 +2351,40 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + + def test_abbreviation(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('--foodle') + parser.add_argument('--foonly') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('bar') + parser1.add_argument('--fo') + parser1.add_argument('--foonew') + + self.assertEqual(parser.parse_args(['--food', 'baz', 'bar']), + NS(foodle='baz', foonly=None, fo=None, foonew=None)) + self.assertEqual(parser.parse_args(['--foon', 'baz', 'bar']), + NS(foodle=None, foonly='baz', fo=None, foonew=None)) + self.assertArgumentParserError(parser.parse_args, ['--fo', 'baz', 'bar']) + self.assertEqual(parser.parse_args(['bar', '--fo', 'baz']), + NS(foodle=None, foonly=None, fo='baz', foonew=None)) + self.assertEqual(parser.parse_args(['bar', '--foo', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertEqual(parser.parse_args(['bar', '--foon', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertArgumentParserError(parser.parse_args, ['bar', '--food', 'baz']) + def test_parse_known_args_with_single_dash_option(self): parser = ErrorRaisingArgumentParser() parser.add_argument('-k', '--known', action='count', default=0) @@ -2714,8 +2787,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2805,11 +2876,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2825,7 +2895,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2835,15 +2905,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2855,7 +2924,7 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) @@ -4586,7 +4655,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -5249,7 +5318,7 @@ def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: + dict(choices=['a', 'b'])]: self.assertTypeError('-x', action=action, **attrs) def test_no_argument_no_const_actions(self): @@ -6495,6 +6564,99 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(res.out.splitlines()[0].decode(), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. RFile.seen = {} diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 1579ad1188d725..0777f39b572486 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -213,53 +213,6 @@ async def run(): self.run_coro(run()) - def test_staggered_race_with_eager_tasks(self): - # See https://github.com/python/cpython/issues/124309 - - async def fail(): - await asyncio.sleep(0) - raise ValueError("no good") - - async def run(): - winner, index, excs = await asyncio.staggered.staggered_race( - [ - lambda: asyncio.sleep(2, result="sleep2"), - lambda: asyncio.sleep(1, result="sleep1"), - lambda: fail() - ], - delay=0.25 - ) - self.assertEqual(winner, 'sleep1') - self.assertEqual(index, 1) - self.assertIsNone(excs[index]) - self.assertIsInstance(excs[0], asyncio.CancelledError) - self.assertIsInstance(excs[2], ValueError) - - self.run_coro(run()) - - def test_staggered_race_with_eager_tasks_no_delay(self): - # See https://github.com/python/cpython/issues/124309 - async def fail(): - raise ValueError("no good") - - async def run(): - winner, index, excs = await asyncio.staggered.staggered_race( - [ - lambda: fail(), - lambda: asyncio.sleep(1, result="sleep1"), - lambda: asyncio.sleep(0, result="sleep0"), - ], - delay=None - ) - self.assertEqual(winner, 'sleep1') - self.assertEqual(index, 1) - self.assertIsNone(excs[index]) - self.assertIsInstance(excs[0], ValueError) - self.assertEqual(len(excs), 2) - - self.run_coro(run()) - - class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = tasks._PyTask diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 21a39b3f911747..e6e32f7dbbbcba 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -82,45 +82,16 @@ async def test_none_successful(self): async def coro(index): raise ValueError(index) - for delay in [None, 0, 0.1, 1]: - with self.subTest(delay=delay): - winner, index, excs = await staggered_race( - [ - lambda: coro(0), - lambda: coro(1), - ], - delay=delay, - ) - - self.assertIs(winner, None) - self.assertIs(index, None) - self.assertEqual(len(excs), 2) - self.assertIsInstance(excs[0], ValueError) - self.assertIsInstance(excs[1], ValueError) - - async def test_long_delay_early_failure(self): - async def coro(index): - await asyncio.sleep(0) # Dummy coroutine for the 1 case - if index == 0: - await asyncio.sleep(0.1) # Dummy coroutine - raise ValueError(index) - - return f'Res: {index}' - winner, index, excs = await staggered_race( [ lambda: coro(0), lambda: coro(1), ], - delay=10, + delay=None, ) - self.assertEqual(winner, 'Res: 1') - self.assertEqual(index, 1) + self.assertIs(winner, None) + self.assertIs(index, None) self.assertEqual(len(excs), 2) self.assertIsInstance(excs[0], ValueError) - self.assertIsNone(excs[1]) - - -if __name__ == "__main__": - unittest.main() + self.assertIsInstance(excs[1], ValueError) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 2ea97e797a4892..d884f54940b471 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2607,6 +2607,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2685,6 +2686,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 1f9ffc5e9a5c33..f119d89c0ec39a 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -985,7 +985,7 @@ def assertFailure(self, *args): def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index bd521a509d07ec..85491a89947318 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -1,13 +1,20 @@ -import unittest +import codecs +import contextlib +import io +import re import sys +import unittest +import unittest.mock as mock +import _testcapi from test.support import import_helper _testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None +BAD_ARGUMENT = re.escape('bad argument type for built-in operation') -class CAPITest(unittest.TestCase): +class CAPIUnicodeTest(unittest.TestCase): # TODO: Test the following functions: # # PyUnicode_BuildEncodingMap @@ -516,5 +523,291 @@ def test_asrawunicodeescapestring(self): # CRASHES asrawunicodeescapestring(NULL) +class CAPICodecs(unittest.TestCase): + + def setUp(self): + # Encoding names are normalized internally by converting them + # to lowercase and their hyphens are replaced by underscores. + self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' + # Make sure that our custom codec is not already registered (that + # way we know whether we correctly unregistered the custom codec + # after a test or not). + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + # create the search function without registering yet + self._create_custom_codec() + + def _create_custom_codec(self): + def codec_encoder(m, errors='strict'): + return (type(m)().join(reversed(m)), len(m)) + + def codec_decoder(c, errors='strict'): + return (type(c)().join(reversed(c)), len(c)) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codec_encoder(input) + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codec_decoder(input) + + class StreamReader(codecs.StreamReader): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + info = codecs.CodecInfo( + encode=codec_encoder, + decode=codec_decoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + name=self.encoding_name + ) + + def search_function(encoding): + if encoding == self.encoding_name: + return info + return None + + self.codec_info = info + self.search_function = search_function + + @contextlib.contextmanager + def use_custom_encoder(self): + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + codecs.register(self.search_function) + yield + codecs.unregister(self.search_function) + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + + def test_codec_register(self): + search_function, encoding = self.search_function, self.encoding_name + # register the search function using the C API + self.assertIsNone(_testcapi.codec_register(search_function)) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIs(codecs.lookup(encoding), search_function(encoding)) + self.assertEqual(codecs.encode('123', encoding=encoding), '321') + # unregister the search function using the regular API + codecs.unregister(search_function) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_unregister(self): + search_function, encoding = self.search_function, self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding) + # register the search function using the regular API + codecs.register(search_function) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIsNotNone(codecs.lookup(encoding)) + # unregister the search function using the C API + self.assertIsNone(_testcapi.codec_unregister(search_function)) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_known_encoding(self): + self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') + self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) + self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) + self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) + + encoding_name = self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding_name) + + codecs.register(self.search_function) + self.addCleanup(codecs.unregister, self.search_function) + + for name in [ + encoding_name, + encoding_name.upper(), + encoding_name.replace('_', '-'), + ]: + with self.subTest(name): + self.assertTrue(_testcapi.codec_known_encoding(name)) + + def test_codec_encode(self): + encode = _testcapi.codec_encode + self.assertEqual(encode('a', 'utf-8', NULL), b'a') + self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') + self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') + + self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + encode('a', NULL, 'strict') + + def test_codec_decode(self): + decode = _testcapi.codec_decode + + s = 'a\xa1\u4f60\U0001f600' + b = s.encode() + + self.assertEqual(decode(b, 'utf-8', 'strict'), s) + self.assertEqual(decode(b, 'utf-8', NULL), s) + self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) + self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) + + # _codecs.decode() only reports an unknown error handling name when + # the corresponding error handling function is used; this difers + # from PyUnicode_Decode() which checks that both the encoding and + # the error handling name are recognized before even attempting to + # call the decoder. + self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') + self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') + + self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + decode(b, NULL, 'strict') + + def test_codec_encoder(self): + codec_encoder = _testcapi.codec_encoder + + with self.use_custom_encoder(): + encoder = codec_encoder(self.encoding_name) + self.assertIs(encoder, self.codec_info.encode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_encoder(NULL) + + def test_codec_decoder(self): + codec_decoder = _testcapi.codec_decoder + + with self.use_custom_encoder(): + decoder = codec_decoder(self.encoding_name) + self.assertIs(decoder, self.codec_info.decode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_decoder(NULL) + + def test_codec_incremental_encoder(self): + codec_incremental_encoder = _testcapi.codec_incremental_encoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + encoder = codec_incremental_encoder(encoding, errors) + self.assertIsInstance(encoder, self.codec_info.incrementalencoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_encoder(NULL, 'strict') + + def test_codec_incremental_decoder(self): + codec_incremental_decoder = _testcapi.codec_incremental_decoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + decoder = codec_incremental_decoder(encoding, errors) + self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_decoder(NULL, 'strict') + + def test_codec_stream_reader(self): + codec_stream_reader = _testcapi.codec_stream_reader + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_reader(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamreader) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_reader(NULL, stream, 'strict') + + def test_codec_stream_writer(self): + codec_stream_writer = _testcapi.codec_stream_writer + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_writer(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamwriter) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_writer(NULL, stream, 'strict') + + +class CAPICodecErrors(unittest.TestCase): + + def test_codec_register_error(self): + # for cleaning up between tests + from _codecs import _unregister_error as _codecs_unregister_error + + self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') + + def custom_error_handler(exc): + raise exc + + error_handler = mock.Mock(wraps=custom_error_handler) + _testcapi.codec_register_error('custom', error_handler) + self.addCleanup(_codecs_unregister_error, 'custom') + + self.assertRaises(UnicodeEncodeError, codecs.encode, + '\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + error_handler.reset_mock() + + self.assertRaises(UnicodeDecodeError, codecs.decode, + b'\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + + # _codecs._unregister_error directly delegates to the internal C + # function so a Python-level function test is sufficient (it is + # tested in test_codeccallbacks). + + def test_codec_lookup_error(self): + codec_lookup_error = _testcapi.codec_lookup_error + self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) + self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) + self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) + self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) + self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) + self.assertRaises(LookupError, codec_lookup_error, 'unknown') + + def test_codec_error_handlers(self): + exceptions = [ + # A UnicodeError with an empty message currently crashes: + # See: https://github.com/python/cpython/issues/123378 + # UnicodeEncodeError('bad', '', 0, 1, 'reason'), + UnicodeEncodeError('bad', 'x', 0, 1, 'reason'), + UnicodeEncodeError('bad', 'xyz123', 0, 1, 'reason'), + UnicodeEncodeError('bad', 'xyz123', 1, 4, 'reason'), + ] + + strict_handler = _testcapi.codec_strict_errors + for exc in exceptions: + with self.subTest(handler=strict_handler, exc=exc): + self.assertRaises(UnicodeEncodeError, strict_handler, exc) + + for handler in [ + _testcapi.codec_ignore_errors, + _testcapi.codec_replace_errors, + _testcapi.codec_xmlcharrefreplace_errors, + _testlimitedcapi.codec_namereplace_errors, + ]: + for exc in exceptions: + with self.subTest(handler=handler, exc=exc): + self.assertIsInstance(handler(exc), tuple) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 01637e1cb7b6e5..71fb9ae45c7c30 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -68,7 +68,7 @@ def test_config_get(self): ("parser_debug", bool, None), ("parse_argv", bool, None), ("pathconfig_warnings", bool, None), - ("perf_profiling", bool, None), + ("perf_profiling", int, None), ("platlibdir", str, "platlibdir"), ("prefix", str | None, "prefix"), ("program_name", str, None), diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 902f788edc22f0..b88c4d16ba4ef4 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -932,6 +933,36 @@ class C: C.a = X() C.a = X() + @cpython_only + def test_detach_materialized_dict_no_memory(self): + # Skip test if _testcapi is not available: + import_helper.import_module('_testcapi') + + code = """if 1: + import test.support + import _testcapi + + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + assert ex.unraisable.exc_type is MemoryError + try: + d["a"] + except KeyError: + pass + else: + assert False, "KeyError not raised" + """ + rc, out, err = script_helper.assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertFalse(out, msg=out.decode('utf-8')) + self.assertFalse(err, msg=err.decode('utf-8')) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4991330489d139..86e5e5c1474674 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,3 +1,4 @@ +from _codecs import _unregister_error as _codecs_unregister_error import codecs import html.entities import itertools @@ -1210,7 +1211,6 @@ def replace_with_long(exc): '\ufffd\x00\x00' ) - def test_fake_error_class(self): handlers = [ codecs.strict_errors, @@ -1235,6 +1235,31 @@ class FakeUnicodeError(Exception): with self.assertRaises((TypeError, FakeUnicodeError)): handler(FakeUnicodeError()) + def test_reject_unregister_builtin_error_handler(self): + for name in [ + 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', + 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', + ]: + with self.subTest(name): + self.assertRaises(ValueError, _codecs_unregister_error, name) + + def test_unregister_custom_error_handler(self): + def custom_handler(exc): + raise exc + + custom_name = 'test.test_unregister_custom_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + codecs.register_error(custom_name, custom_handler) + self.assertIs(codecs.lookup_error(custom_name), custom_handler) + self.assertTrue(_codecs_unregister_error(custom_name)) + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + + def test_unregister_custom_unknown_error_handler(self): + unknown_name = 'test.test_unregister_custom_unknown_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + self.assertFalse(_codecs_unregister_error(unknown_name)) + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index b81d847c824273..e9ee72cf234fdc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import contextlib import dis import io +import itertools import math import opcode import os @@ -12,7 +13,10 @@ import types import textwrap import warnings -import _testinternalcapi +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test import support from test.support import (script_helper, requires_debug_ranges, run_code, @@ -2644,6 +2648,8 @@ def test_return_inside_async_with_block(self): """ self.check_stack_size(snippet, async_=True) +@support.cpython_only +@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class TestInstructionSequence(unittest.TestCase): def compare_instructions(self, seq, expected): self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], @@ -2687,6 +2693,22 @@ def test_nested(self): self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + def test_static_attributes_are_sorted(self): + code = ( + 'class T:\n' + ' def __init__(self):\n' + ' self.{V1} = 10\n' + ' self.{V2} = 10\n' + ' def foo(self):\n' + ' self.{V3} = 10\n' + ) + attributes = ("a", "b", "c") + for perm in itertools.permutations(attributes): + var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} + ns = run_code(code.format(**var_names)) + t = ns['T'] + self.assertEqual(t.__static_attributes__, attributes) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 69e86162e0c11a..2e6c49e29ce828 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -17,6 +17,7 @@ from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict +from copy import deepcopy from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. @@ -60,7 +61,7 @@ class C: x: int = field(default=1, default_factory=int) def test_field_repr(self): - int_field = field(default=1, init=True, repr=False) + int_field = field(default=1, init=True, repr=False, doc='Docstring') int_field.name = "id" repr_output = repr(int_field) expected_output = "Field(name='id',type=None," \ @@ -68,6 +69,7 @@ def test_field_repr(self): "init=True,repr=False,hash=None," \ "compare=True,metadata=mappingproxy({})," \ f"kw_only={MISSING!r}," \ + "doc='Docstring'," \ "_field_type=None)" self.assertEqual(repr_output, expected_output) @@ -3175,6 +3177,48 @@ class C: with self.assertRaisesRegex(TypeError, 'unhashable type'): hash(C({})) + def test_frozen_deepcopy_without_slots(self): + # see: https://github.com/python/cpython/issues/89683 + @dataclass(frozen=True, slots=False) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + def test_frozen_deepcopy_with_slots(self): + # see: https://github.com/python/cpython/issues/89683 + with self.subTest('generated __slots__'): + @dataclass(frozen=True, slots=True) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + with self.subTest('user-defined __slots__ and no __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + s: str + + # with user-defined slots, __getstate__ and __setstate__ are not + # automatically added, hence the error + err = r"^cannot\ assign\ to\ field\ 's'$" + self.assertRaisesRegex(FrozenInstanceError, err, deepcopy, C('')) + + with self.subTest('user-defined __slots__ and __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + __getstate__ = dataclasses._dataclass_getstate + __setstate__ = dataclasses._dataclass_setstate + + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + class TestSlots(unittest.TestCase): def test_simple(self): @@ -3261,7 +3305,7 @@ class Base(Root4): j: str h: str - self.assertEqual(Base.__slots__, ('y', )) + self.assertEqual(Base.__slots__, ('y',)) @dataclass(slots=True) class Derived(Base): @@ -3271,7 +3315,7 @@ class Derived(Base): k: str h: str - self.assertEqual(Derived.__slots__, ('z', )) + self.assertEqual(Derived.__slots__, ('z',)) @dataclass class AnotherDerived(Base): @@ -3279,6 +3323,24 @@ class AnotherDerived(Base): self.assertNotIn('__slots__', AnotherDerived.__dict__) + def test_slots_with_docs(self): + class Root: + __slots__ = {'x': 'x'} + + @dataclass(slots=True) + class Base(Root): + y1: int = field(doc='y1') + y2: int + + self.assertEqual(Base.__slots__, {'y1': 'y1', 'y2': None}) + + @dataclass(slots=True) + class Child(Base): + z1: int = field(doc='z1') + z2: int + + self.assertEqual(Child.__slots__, {'z1': 'z1', 'z2': None}) + def test_cant_inherit_from_iterator_slots(self): class Root: @@ -4255,6 +4317,23 @@ def test_funny_class_names_names(self): C = make_dataclass(classname, ['a', 'b']) self.assertEqual(C.__name__, classname) + def test_dataclass_decorator_default(self): + C = make_dataclass('C', [('x', int)], decorator=dataclass) + c = C(10) + self.assertEqual(c.x, 10) + + def test_dataclass_custom_decorator(self): + def custom_dataclass(cls, *args, **kwargs): + dc = dataclass(cls, *args, **kwargs) + dc.__custom__ = True + return dc + + C = make_dataclass('C', [('x', int)], decorator=custom_dataclass) + c = C(10) + self.assertEqual(c.x, 10) + self.assertEqual(c.__custom__, True) + + class TestReplace(unittest.TestCase): def test(self): @dataclass(frozen=True) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 12479e32d0f5db..c591fd54430b18 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4381,7 +4381,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): diff --git a/Lib/test/test_dict_version.py b/Lib/test/test_dict_version.py deleted file mode 100644 index 243084c75c42bc..00000000000000 --- a/Lib/test/test_dict_version.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Test implementation of the PEP 509: dictionary versioning. -""" -import unittest -from test.support import import_helper - -# PEP 509 is implemented in CPython but other Python implementations -# don't require to implement it -_testcapi = import_helper.import_module('_testcapi') - - -class DictVersionTests(unittest.TestCase): - type2test = dict - - def setUp(self): - self.seen_versions = set() - self.dict = None - - def check_version_unique(self, mydict): - version = _testcapi.dict_get_version(mydict) - self.assertNotIn(version, self.seen_versions) - self.seen_versions.add(version) - - def check_version_changed(self, mydict, method, *args, **kw): - result = method(*args, **kw) - self.check_version_unique(mydict) - return result - - def check_version_dont_change(self, mydict, method, *args, **kw): - version1 = _testcapi.dict_get_version(mydict) - self.seen_versions.add(version1) - - result = method(*args, **kw) - - version2 = _testcapi.dict_get_version(mydict) - self.assertEqual(version2, version1, "version changed") - - return result - - def new_dict(self, *args, **kw): - d = self.type2test(*args, **kw) - self.check_version_unique(d) - return d - - def test_constructor(self): - # new empty dictionaries must all have an unique version - empty1 = self.new_dict() - empty2 = self.new_dict() - empty3 = self.new_dict() - - # non-empty dictionaries must also have an unique version - nonempty1 = self.new_dict(x='x') - nonempty2 = self.new_dict(x='x', y='y') - - def test_copy(self): - d = self.new_dict(a=1, b=2) - - d2 = self.check_version_dont_change(d, d.copy) - - # dict.copy() must create a dictionary with a new unique version - self.check_version_unique(d2) - - def test_setitem(self): - d = self.new_dict() - - # creating new keys must change the version - self.check_version_changed(d, d.__setitem__, 'x', 'x') - self.check_version_changed(d, d.__setitem__, 'y', 'y') - - # changing values must change the version - self.check_version_changed(d, d.__setitem__, 'x', 1) - self.check_version_changed(d, d.__setitem__, 'y', 2) - - def test_setitem_same_value(self): - value = object() - d = self.new_dict() - - # setting a key must change the version - self.check_version_changed(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.__setitem__ - # must change the version - self.check_version_dont_change(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.update - # must change the version - self.check_version_dont_change(d, d.update, key=value) - - d2 = self.new_dict(key=value) - self.check_version_dont_change(d, d.update, d2) - - def test_setitem_equal(self): - class AlwaysEqual: - def __eq__(self, other): - return True - - value1 = AlwaysEqual() - value2 = AlwaysEqual() - self.assertTrue(value1 == value2) - self.assertFalse(value1 != value2) - self.assertIsNot(value1, value2) - - d = self.new_dict() - self.check_version_changed(d, d.__setitem__, 'key', value1) - self.assertIs(d['key'], value1) - - # setting a key to a value equal to the current value - # with dict.__setitem__() must change the version - self.check_version_changed(d, d.__setitem__, 'key', value2) - self.assertIs(d['key'], value2) - - # setting a key to a value equal to the current value - # with dict.update() must change the version - self.check_version_changed(d, d.update, key=value1) - self.assertIs(d['key'], value1) - - d2 = self.new_dict(key=value2) - self.check_version_changed(d, d.update, d2) - self.assertIs(d['key'], value2) - - def test_setdefault(self): - d = self.new_dict() - - # setting a key with dict.setdefault() must change the version - self.check_version_changed(d, d.setdefault, 'key', 'value1') - - # don't change the version if the key already exists - self.check_version_dont_change(d, d.setdefault, 'key', 'value2') - - def test_delitem(self): - d = self.new_dict(key='value') - - # deleting a key with dict.__delitem__() must change the version - self.check_version_changed(d, d.__delitem__, 'key') - - # don't change the version if the key doesn't exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.__delitem__, 'key') - - def test_pop(self): - d = self.new_dict(key='value') - - # pop() must change the version if the key exists - self.check_version_changed(d, d.pop, 'key') - - # pop() must not change the version if the key does not exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.pop, 'key') - - def test_popitem(self): - d = self.new_dict(key='value') - - # popitem() must change the version if the dict is not empty - self.check_version_changed(d, d.popitem) - - # popitem() must not change the version if the dict is empty - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.popitem) - - def test_update(self): - d = self.new_dict(key='value') - - # update() calling with no argument must not change the version - self.check_version_dont_change(d, d.update) - - # update() must change the version - self.check_version_changed(d, d.update, key='new value') - - d2 = self.new_dict(key='value 3') - self.check_version_changed(d, d.update, d2) - - def test_clear(self): - d = self.new_dict(key='value') - - # clear() must change the version if the dict is not empty - self.check_version_changed(d, d.clear) - - # clear() must not change the version if the dict is empty - self.check_version_dont_change(d, d.clear) - - -class Dict(dict): - pass - - -class DictSubtypeVersionTests(DictVersionTests): - type2test = Dict - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7c5cb855a397ab..3edc19d8254754 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -560,7 +560,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'cpu_count': -1, 'faulthandler': False, 'tracemalloc': 0, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': False, 'code_debug_ranges': True, 'show_ref_count': False, @@ -652,7 +652,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): use_hash_seed=False, faulthandler=False, tracemalloc=False, - perf_profiling=False, + perf_profiling=0, pathconfig_warnings=False, ) if MS_WINDOWS: @@ -966,7 +966,7 @@ def test_init_from_config(self): 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'show_ref_count': True, @@ -1031,7 +1031,7 @@ def test_init_compat_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1051,6 +1051,7 @@ def test_init_compat_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = 1 @@ -1066,7 +1067,7 @@ def test_init_python_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1086,6 +1087,7 @@ def test_init_python_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = True @@ -1763,6 +1765,7 @@ def test_initconfig_api(self): 'xoptions': {'faulthandler': True}, 'hash_seed': 10, 'use_hash_seed': True, + 'perf_profiling': 2, } config_dev_mode(preconfig, config) self.check_all_configs("test_initconfig_api", config, preconfig, diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 3126458e08e50a..80daf0d9cae9e0 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -142,41 +142,6 @@ def writer_func(l): for ref in thread_list: self.assertIsNone(ref()) - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_dict_version(self): - dict_version = _testcapi.dict_version - THREAD_COUNT = 10 - DICT_COUNT = 10000 - lists = [] - writers = [] - - def writer_func(thread_list): - for i in range(DICT_COUNT): - thread_list.append(dict_version({})) - - for x in range(THREAD_COUNT): - thread_list = [] - lists.append(thread_list) - writer = Thread(target=partial(writer_func, thread_list)) - writers.append(writer) - - for writer in writers: - writer.start() - - for writer in writers: - writer.join() - - total_len = 0 - values = set() - for thread_list in lists: - for v in thread_list: - if v in values: - print('dup', v, (v/4096)%256) - values.add(v) - total_len += len(thread_list) - versions = set(dict_version for thread_list in lists for dict_version in thread_list) - self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 906f9884d6792f..bb7df1f5cfa7f7 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1048,6 +1048,24 @@ class Z: callback.assert_not_called() gc.enable() + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + class IncrementalGCTests(unittest.TestCase): diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43e9f852022934..43fda6622537fc 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -357,3 +357,15 @@ class td354(typing.TypedDict): # line 358 td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index aeee504fb8b555..d2dc9e147d29c2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -835,6 +835,47 @@ class C: nonlocal __firstlineno__ self.assertRaises(OSError, inspect.getsource, C) +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -947,6 +988,11 @@ def test_typeddict(self): self.assertSourceEqual(mod2.td354, 354, 356) self.assertRaises(OSError, inspect.getsource, mod2.td359) + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1010,7 +1056,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -1018,9 +1064,20 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 351f1067c10343..2a326684460b99 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -46,10 +46,14 @@ def nth_line(func, offset): class MonitoringBasicTest(unittest.TestCase): + def tearDown(self): + sys.monitoring.free_tool_id(TEST_TOOL) + def test_has_objects(self): m = sys.monitoring m.events m.use_tool_id + m.clear_tool_id m.free_tool_id m.get_tool m.get_events @@ -77,6 +81,43 @@ def test_tool(self): with self.assertRaises(ValueError): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + def test_clear(self): + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, lambda *args: events.append(args)) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + def f(): + a = 1 + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + + f() + sys.monitoring.clear_tool_id(TEST_TOOL) + f() + + # the first f() should trigger a PY_START and a LINE event + # the second f() after clear_tool_id should not trigger any event + # the callback function should be cleared as well + self.assertEqual(len(events), 2) + callback = sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertIs(callback, None) + + sys.monitoring.free_tool_id(TEST_TOOL) + + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + f() + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + f() + # the first f() should trigger a LINE event, and even if we use the + # tool id immediately after freeing it, the second f() should not + # trigger any event + self.assertEqual(len(events), 1) + sys.monitoring.free_tool_id(TEST_TOOL) + class MonitoringTestBase: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 84c0e1073a1054..4c64a800cb32d2 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -363,6 +363,54 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_commands(): + """Test the commands command of pdb. + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... print(1) + ... print(2) + ... print(3) + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'commands', + ... 'silent', # suppress the frame status output + ... 'p "hello"', + ... 'end', + ... 'b 4', + ... 'commands', + ... 'until 5', # no output, should stop at line 5 + ... 'continue', # hit breakpoint at line 3 + ... '', # repeat continue, hit breakpoint at line 4 then `until` to line 5 + ... '', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) commands + (com) silent + (com) p "hello" + (com) end + (Pdb) b 4 + Breakpoint 2 at :4 + (Pdb) commands + (com) until 5 + (Pdb) continue + 'hello' + (Pdb) + 1 + 2 + > (5)test_function() + -> print(3) + (Pdb) + 3 + """ + def test_pdb_breakpoint_with_filename(): """Breakpoints with filename:lineno diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index b68a55259c62e1..672851425ffb53 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -23,6 +23,15 @@ raise unittest.SkipTest("test crash randomly on ASAN/MSAN/UBSAN build") +def is_jit_build(): + cflags = (sysconfig.get_config_var("PY_CORE_CFLAGS") or '') + return "_Py_JIT" in cflags + + +if is_jit_build(): + raise unittest.SkipTest("Perf support is not available in JIT builds") + + def supports_trampoline_profiling(): perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") if not perf_trampoline: @@ -229,7 +238,7 @@ def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False - return "no-omit-frame-pointer" in cflags and "_Py_JIT" not in cflags + return "no-omit-frame-pointer" in cflags def perf_command_works(): @@ -382,6 +391,7 @@ def baz(n): self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) + @unittest.skipUnless(perf_command_works(), "perf command doesn't work") @unittest.skipUnless( is_unwinding_reliable_with_frame_pointers(), @@ -494,7 +504,9 @@ def _is_perf_version_at_least(major, minor): @unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(_is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug") +@unittest.skipUnless( + _is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug" +) class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): def run_perf(self, script_dir, script, activate_trampoline=True): if activate_trampoline: diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2dba077cdea6a7..776e02f41a1cec 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -463,6 +463,14 @@ class BinaryInteger(enum.IntEnum): doc = pydoc.render_doc(BinaryInteger) self.assertIn('BinaryInteger.zero', doc) + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + def test_mixed_case_module_names_are_lower_cased(self): # issue16484 doc_link = get_pydoc_link(xml.etree.ElementTree) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 0f3e9996e77e45..36f940eaea4eac 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1080,7 +1080,7 @@ def setUp(self): @force_not_colorized def test_exposed_globals_in_repl(self): - pre = "['__annotations__', '__builtins__'" + pre = "['__builtins__'" post = "'__loader__', '__name__', '__package__', '__spec__']" output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) if "can't use pyrepl" in output: diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 7a7285a1a2fcfd..e764e60560db23 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -291,5 +291,42 @@ def f(): self.assertEqual(traceback_lines, expected_lines) +class TestAsyncioREPLContextVars(unittest.TestCase): + def test_toplevel_contextvars_sync(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar("var", default="failed") + var.set("ok") + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = dedent(""" + print(f"toplevel contextvar test: {var.get()}") + """) + p.stdin.write(user_input2) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + def test_toplevel_contextvars_async(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar('var', default='failed') + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = "async def set_var(): var.set('ok')\n" + p.stdin.write(user_input2) + user_input3 = "await set_var()\n" + p.stdin.write(user_input3) + user_input4 = "print(f'toplevel contextvar test: {var.get()}')\n" + p.stdin.write(user_input4) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e60e5477d32e1f..9a3cf140d81241 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -548,13 +548,14 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 293799ff68ea05..530c317a852e77 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -654,8 +654,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 9a04a95dc40d65..eeaf5de2e303f6 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -227,13 +227,13 @@ def test_element_create_image(self): foreground='blue', background='yellow') img3 = tkinter.BitmapImage(master=self.root, file=imgfile, foreground='white', background='black') - style.element_create('Button.button', 'image', + style.element_create('TestButton.button', 'image', img1, ('pressed', img2), ('active', img3), border=(2, 4), sticky='we') - self.assertIn('Button.button', style.element_names()) + self.assertIn('TestButton.button', style.element_names()) - style.layout('Button', [('Button.button', {'sticky': 'news'})]) - b = ttk.Button(self.root, style='Button') + style.layout('TestButton', [('TestButton.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='TestButton') b.pack(expand=True, fill='both') self.assertEqual(b.winfo_reqwidth(), 16) self.assertEqual(b.winfo_reqheight(), 16) diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 49d6aa810304fb..ebb65d8c6cf81b 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -211,6 +211,19 @@ def test_generic(self): self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] def test_keywords(self): TA = TypeAliasType(name="TA", value=int) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 91082e6b23c04b..257b7fa95dcb76 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -375,7 +375,7 @@ class X: with self.assertRaises(NotImplementedError): annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): - annotate(annotationlib.Format.SOURCE) + annotate(annotationlib.Format.STRING) with self.assertRaises(NotImplementedError): annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 8c21553e410d8a..433b19593bdd04 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1440,7 +1440,7 @@ def f[T: int = int, **P = int, *Ts = int](): pass self.assertIs(case(1), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) - self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.SOURCE), 'int') + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') def test_constraints(self): def f[T: (int, str)](): pass @@ -1451,7 +1451,7 @@ def f[T: (int, str)](): pass self.assertEqual(case.evaluate_constraints(1), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) - self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.SOURCE), '(int, str)') + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') def test_const_evaluator(self): T = TypeVar("T", bound=int) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3ac6b97383fcef..2f1f9e86a0bce4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7059,7 +7059,7 @@ class C: self.assertIsInstance(annos['x'], annotationlib.ForwardRef) self.assertEqual(annos['x'].__arg__, 'undefined') - self.assertEqual(get_type_hints(C, format=annotationlib.Format.SOURCE), + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), {'x': 'undefined'}) @@ -7898,7 +7898,7 @@ class Z(NamedTuple): self.assertEqual(Z.__annotations__, annos) self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Z.__annotate__(annotationlib.Format.SOURCE), {"a": "None", "b": "str"}) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) def test_future_annotations(self): code = """ @@ -8241,7 +8241,7 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__annotations__, annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Emp.__annotate__(annotationlib.Format.SOURCE), {'name': 'str', 'id': 'int'}) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8603,7 +8603,7 @@ class A[T](TypedDict): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8616,7 +8616,7 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8628,7 +8628,7 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) self.assertEqual(A2.__annotations__, {'a': T}) - self.assertEqual(A2.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8640,7 +8640,7 @@ class B(A[KT], total=False): self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) - self.assertEqual(B.__annotate__(annotationlib.Format.SOURCE), {'a': 'T', 'b': 'KT'}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8665,7 +8665,7 @@ class C(B[int]): 'b': KT, 'c': int, }) - self.assertEqual(C.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8689,7 +8689,7 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) - self.assertEqual(Point3D.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'T', 'c': 'KT', @@ -8725,7 +8725,7 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) - self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8929,7 +8929,7 @@ class A(TypedDict): A.__annotations__ self.assertEqual( - A.__annotate__(annotationlib.Format.SOURCE), + A.__annotate__(annotationlib.Format.STRING), {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', 'z': 'Required[undefined]'}, ) diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index c9c20f008ca5a2..f260769eb8c35e 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -8,8 +8,10 @@ Mock, ANY, _CallList, patch, PropertyMock, _callable ) +from dataclasses import dataclass, field, InitVar from datetime import datetime from functools import partial +from typing import ClassVar class SomeClass(object): def one(self, a, b): pass @@ -1034,6 +1036,76 @@ def f(a): pass self.assertEqual(mock.mock_calls, []) self.assertEqual(rv.mock_calls, []) + def test_dataclass_post_init(self): + @dataclass + class WithPostInit: + a: int = field(init=False) + b: int = field(init=False) + def __post_init__(self): + self.a = 1 + self.b = 2 + + for mock in [ + create_autospec(WithPostInit, instance=True), + create_autospec(WithPostInit()), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + # Classes do not have these fields: + mock = create_autospec(WithPostInit) + msg = "Mock object has no attribute" + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b + + def test_dataclass_default(self): + @dataclass + class WithDefault: + a: int + b: int = 0 + + for mock in [ + create_autospec(WithDefault, instance=True), + create_autospec(WithDefault(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + def test_dataclass_with_method(self): + @dataclass + class WithMethod: + a: int + def b(self) -> int: + return 1 + + for mock in [ + create_autospec(WithMethod, instance=True), + create_autospec(WithMethod(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + mock.b.assert_not_called() + + def test_dataclass_with_non_fields(self): + @dataclass + class WithNonFields: + a: ClassVar[int] + b: InitVar[int] + + msg = "Mock object has no attribute" + for mock in [ + create_autospec(WithNonFields, instance=True), + create_autospec(WithNonFields(1)), + ]: + with self.subTest(mock=mock): + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b class TestCallList(unittest.TestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 252eef32cd88a4..c924c767042552 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -245,7 +245,7 @@ def _type_repr(obj): if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return annotationlib.value_to_source(obj) + return annotationlib.value_to_string(obj) def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): @@ -1036,7 +1036,7 @@ def evaluate_forward_ref( * Recursively evaluates forward references nested within the type hint. * Rejects certain objects that are not valid type hints. * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *SOURCE* formats. + * Supports the *FORWARDREF* and *STRING* formats. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -1053,7 +1053,7 @@ def evaluate_forward_ref( if type_params is _sentinel: _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") type_params = () - if format == annotationlib.Format.SOURCE: + if format == annotationlib.Format.STRING: return forward_ref.__forward_arg__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref @@ -2380,7 +2380,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, hints = {} for base in reversed(obj.__mro__): ann = annotationlib.get_annotations(base, format=format) - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: hints.update(ann) continue if globalns is None: @@ -2404,7 +2404,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, value = _eval_type(value, base_globals, base_locals, base.__type_params__, format=format, owner=obj) hints[name] = value - if include_extras or format is annotationlib.Format.SOURCE: + if include_extras or format is annotationlib.Format.STRING: return hints else: return {k: _strip_annotations(t) for k, t in hints.items()} @@ -2418,7 +2418,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, and not hasattr(obj, '__annotate__') ): raise TypeError(f"{obj!r} is not a module, class, or callable.") - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: return hints if globalns is None: @@ -2937,7 +2937,7 @@ def annotate(format): if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): return checked_types else: - return annotationlib.annotations_to_source(types) + return annotationlib.annotations_to_string(types) return annotate @@ -2972,7 +2972,7 @@ def __new__(cls, typename, bases, ns): def annotate(format): annos = annotationlib.call_annotate_function(original_annotate, format) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: return {key: _type_check(val, f"field {key} annotation must be a type") for key, val in annos.items()} return annos @@ -3220,13 +3220,13 @@ def __annotate__(format): annos.update(base_annos) if own_annotate is not None: own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: own = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own.items() } - elif format == annotationlib.Format.SOURCE: - own = annotationlib.annotations_to_source(own_annotations) + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) else: own = own_checked_annotations annos.update(own) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index bb34c7436047ad..21ca061a77c26f 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -34,6 +34,7 @@ import pkgutil from inspect import iscoroutinefunction import threading +from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -628,7 +629,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None, *, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -2218,7 +2221,7 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() - def reset_mock(self, /, *args, return_value=False, **kwargs): + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): if ( return_value and self._mock_name @@ -2754,7 +2757,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) - _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + _kwargs = {'spec': [f.name for f in dataclass_fields]} + else: + _kwargs = {'spec': spec} + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2811,7 +2822,7 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, _name='()', _parent=mock, wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2825,10 +2836,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue child_kwargs = {'spec': original} # Wrap child attributes also. diff --git a/Makefile.pre.in b/Makefile.pre.in index a4d99262702a17..07c8a4d20142db 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -490,7 +490,7 @@ PYTHON_OBJS= \ Python/thread.o \ Python/traceback.o \ Python/tracemalloc.o \ - Python/typeid.o \ + Python/uniqueid.o \ Python/getopt.o \ Python/pystrcmp.o \ Python/pystrtod.o \ @@ -1279,7 +1279,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ - $(srcdir)/Include/internal/pycore_typeid.h \ + $(srcdir)/Include/internal/pycore_uniqueid.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ diff --git a/Misc/ACKS b/Misc/ACKS index b2529601a2f71a..d94cbacf888468 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1552,6 +1552,7 @@ Lisa Roach Carl Robben Ben Roberts Mark Roberts +Tony Roberts Andy Robinson Izan "TizzySaurus" Robinson Jim Robinson diff --git a/Misc/NEWS.d/next/C_API/2024-09-24-20-34-21.gh-issue-124296.S4QoS1.rst b/Misc/NEWS.d/next/C_API/2024-09-24-20-34-21.gh-issue-124296.S4QoS1.rst new file mode 100644 index 00000000000000..e7b9187655eb31 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-24-20-34-21.gh-issue-124296.S4QoS1.rst @@ -0,0 +1,3 @@ +:c:type:`PyDictObject` no longer maintains a private version tag field +``ma_version_tag`` per :pep:`699`. This field was originally added in +Python 3.6 (:pep:`509`) and deprecated in Python 3.12. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst new file mode 100644 index 00000000000000..de62875e16475d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst @@ -0,0 +1,2 @@ +Improved JIT memory consumption by periodically freeing memory used by infrequently-executed code. +This change is especially likely to improve the memory footprint of long-running programs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst new file mode 100644 index 00000000000000..25b47d5fbaefa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst @@ -0,0 +1,3 @@ +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst new file mode 100644 index 00000000000000..58e79f22ac0f90 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst @@ -0,0 +1,2 @@ +Fix nondeterminism in compilation by sorting the value of +:attr:`~type.__static_attributes__`. Patch by kp2pml30. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst new file mode 100644 index 00000000000000..1005c651849f45 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst @@ -0,0 +1,3 @@ +When deallocating an object with inline values whose ``__dict__`` is still +live: if memory allocation for the inline values fails, clear the +dictionary. Prevents an interpreter crash. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst new file mode 100644 index 00000000000000..4cdbb205c962c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst @@ -0,0 +1,2 @@ +The ``__main__`` module no longer always contains an ``__annotations__`` +dictionary in its global namespace. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst new file mode 100644 index 00000000000000..fc3f8af72d87bf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst @@ -0,0 +1,3 @@ +Fix a bug that can cause a crash when sub-interpreters use "basic" +single-phase extension modules. Shared objects could refer to PyGC_Head +nodes that had been freed as part of interpreter cleanup. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-17-18-53.gh-issue-124642.OCjhBJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-17-18-53.gh-issue-124642.OCjhBJ.rst new file mode 100644 index 00000000000000..29763844a9f592 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-17-18-53.gh-issue-124642.OCjhBJ.rst @@ -0,0 +1 @@ +Fixed scalability issue in free-threaded builds for lock-free reads from dictionaries in multi-threaded scenarios diff --git a/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst new file mode 100644 index 00000000000000..f9da7b8a5724f5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst @@ -0,0 +1,2 @@ +Remove coverity scan from the CPython repo. It has not been used since 2020 +and is currently unmaintained. diff --git a/Misc/NEWS.d/next/Documentation/2024-09-27-16-47-48.gh-issue-124720.nVSTVb.rst b/Misc/NEWS.d/next/Documentation/2024-09-27-16-47-48.gh-issue-124720.nVSTVb.rst new file mode 100644 index 00000000000000..6bef1e4158400b --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-27-16-47-48.gh-issue-124720.nVSTVb.rst @@ -0,0 +1,2 @@ +Update "Using Python on a Mac" section of the "Python Setup and Usage" +document and include information on installing free-threading support. diff --git a/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst new file mode 100644 index 00000000000000..8e1937ab73c31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst @@ -0,0 +1,9 @@ +Add *doc* parameter to :func:`dataclasses.field`, so it can be stored and +shown as a documentation / metadata. If ``@dataclass(slots=True)`` is used, +then the supplied string is availabl in the :attr:`~object.__slots__` dict. +Otherwise, the supplied string is only available in the corresponding +:class:`dataclasses.Field` object. + +In order to support this feature we are changing the ``__slots__`` format +in dataclasses from :class:`tuple` to :class:`dict` +when documentation / metadata is present. diff --git a/Misc/NEWS.d/next/Library/2024-08-06-07-24-00.gh-issue-118974.qamsCQ.rst b/Misc/NEWS.d/next/Library/2024-08-06-07-24-00.gh-issue-118974.qamsCQ.rst new file mode 100644 index 00000000000000..79480a69c1a90e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-06-07-24-00.gh-issue-118974.qamsCQ.rst @@ -0,0 +1,2 @@ +Add ``decorator`` parameter to :func:`dataclasses.make_dataclass` +to customize the functional creation of dataclasses. diff --git a/Misc/NEWS.d/next/Library/2024-08-23-15-49-10.gh-issue-116810.QLBUU8.rst b/Misc/NEWS.d/next/Library/2024-08-23-15-49-10.gh-issue-116810.QLBUU8.rst new file mode 100644 index 00000000000000..0e5256e7151c5a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-23-15-49-10.gh-issue-116810.QLBUU8.rst @@ -0,0 +1,4 @@ +Resolve a memory leak introduced in CPython 3.10's :mod:`ssl` when the +:attr:`ssl.SSLSocket.session` property was accessed. Speeds up read and +write access to said property by no longer unnecessarily cloning session +objects via serialization. diff --git a/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst new file mode 100644 index 00000000000000..e388541f1c2c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst @@ -0,0 +1,4 @@ +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. diff --git a/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst new file mode 100644 index 00000000000000..a4428e20f3ccdd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst @@ -0,0 +1,5 @@ +The default :mod:`multiprocessing` start method on Linux and other POSIX +systems has been changed away from often unsafe ``"fork"`` to ``"forkserver"`` +(when the platform supports sending file handles over pipes as most do) or +``"spawn"``. Mac and Windows are unchanged as they already default to +``"spawn"``. diff --git a/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst new file mode 100644 index 00000000000000..707c4d651cb5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst @@ -0,0 +1,2 @@ +Fix disallowing abbreviation of single-dash long options in :mod:`argparse` +with ``allow_abbrev=False``. diff --git a/Misc/NEWS.d/next/Library/2024-09-23-18-18-23.gh-issue-124309.iFcarA.rst b/Misc/NEWS.d/next/Library/2024-09-23-18-18-23.gh-issue-124309.iFcarA.rst deleted file mode 100644 index 89610fa44bf743..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-09-23-18-18-23.gh-issue-124309.iFcarA.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed :exc:`AssertionError` when using :func:`!asyncio.staggered.staggered_race` with :attr:`asyncio.eager_task_factory`. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-00-01-24.gh-issue-124400.0XCgfe.rst b/Misc/NEWS.d/next/Library/2024-09-24-00-01-24.gh-issue-124400.0XCgfe.rst new file mode 100644 index 00000000000000..25ee01e3108bf8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-00-01-24.gh-issue-124400.0XCgfe.rst @@ -0,0 +1 @@ +Fixed a :mod:`pdb` bug where ``until`` has no effect when it appears in a ``commands`` sequence. Also avoid printing the frame information at a breakpoint that has a command list containing a command that resumes execution. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst new file mode 100644 index 00000000000000..dff902d8c6139a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst @@ -0,0 +1,2 @@ +:mod:`argparse` vim supports abbreviated single-dash long options separated +by ``=`` from its value. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst new file mode 100644 index 00000000000000..38c030668b6b42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst @@ -0,0 +1,4 @@ +Add support for :func:`dataclasses.dataclass` in +:func:`unittest.mock.create_autospec`. Now ``create_autospec`` will check +for potential dataclasses and use :func:`dataclasses.fields` function to +retrieve the spec information. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst new file mode 100644 index 00000000000000..45fe4786fa6563 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst @@ -0,0 +1,2 @@ +Due to unreliable results on some devices, :func:`time.strftime` no longer +accepts negative years on Android. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst new file mode 100644 index 00000000000000..20ba1534f5e99d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst @@ -0,0 +1,4 @@ +Fix support of arguments with :ref:`choices` in :mod:`argparse`. Positional +arguments with :ref:`nargs` equal to ``'?'`` or ``'*'`` no longer check +:ref:`default` against ``choices``. Optional arguments with ``nargs`` equal +to ``'?'`` no longer check :ref:`const` against ``choices``. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst new file mode 100644 index 00000000000000..4dbf4eb709733d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst @@ -0,0 +1,2 @@ +Fix :class:`typing.TypeAliasType` not to be generic, when ``type_params`` is +an empty tuple. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst new file mode 100644 index 00000000000000..bb451cdd9ae44c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` support of positional arguments with ``nargs='?'``, +``default=argparse.SUPPRESS`` and specified ``type``. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst new file mode 100644 index 00000000000000..33ae037ae56b0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst @@ -0,0 +1 @@ +Fixed crash when using :func:`gc.get_referents` on a capsule object. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-00-35-24.gh-issue-116750.X1aMHI.rst b/Misc/NEWS.d/next/Library/2024-09-26-00-35-24.gh-issue-116750.X1aMHI.rst new file mode 100644 index 00000000000000..cf9dacf4007c28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-00-35-24.gh-issue-116750.X1aMHI.rst @@ -0,0 +1 @@ +Provide :func:`sys.monitoring.clear_tool_id` to unregister all events and callbacks set by the tool. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst new file mode 100644 index 00000000000000..801a5fdd4abd4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst @@ -0,0 +1,2 @@ +Fix support of :ref:`choices` with string value in :mod:`argparse`. Substrings +of the specified string no longer considered valid values. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst b/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst new file mode 100644 index 00000000000000..ac48bd84930745 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst @@ -0,0 +1 @@ +All :mod:`asyncio` REPL prompts run in the same :class:`context `. Contributed by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-22-14-12.gh-issue-58573.hozbm9.rst b/Misc/NEWS.d/next/Library/2024-09-26-22-14-12.gh-issue-58573.hozbm9.rst new file mode 100644 index 00000000000000..37d64ee536ff49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-22-14-12.gh-issue-58573.hozbm9.rst @@ -0,0 +1,2 @@ +Fix conflicts between abbreviated long options in the parent parser and +subparsers in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst new file mode 100644 index 00000000000000..62639a16c52aa0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` for namespaces with not directly writable dict (e.g. +classes). diff --git a/Misc/NEWS.d/next/Library/2024-09-30-19-59-28.gh-issue-66436.4gYN_n.rst b/Misc/NEWS.d/next/Library/2024-09-30-19-59-28.gh-issue-66436.4gYN_n.rst new file mode 100644 index 00000000000000..69a77b01902873 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-30-19-59-28.gh-issue-66436.4gYN_n.rst @@ -0,0 +1,4 @@ +Improved :ref:`prog` default value for :class:`argparse.ArgumentParser`. It +will now include the name of the Python executable along with the module or +package name, or the path to a directory, ZIP file, or directory within a +ZIP file if the code was run that way. diff --git a/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst new file mode 100644 index 00000000000000..203868a8fee39c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst @@ -0,0 +1 @@ +Fix ``_Py_ThreadId`` for Windows builds using MinGW. Patch by Tony Roberts. diff --git a/Misc/NEWS.d/next/Windows/2024-09-27-15-07-30.gh-issue-124487.7LrwHC.rst b/Misc/NEWS.d/next/Windows/2024-09-27-15-07-30.gh-issue-124487.7LrwHC.rst new file mode 100644 index 00000000000000..93fb68d28c702e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-27-15-07-30.gh-issue-124487.7LrwHC.rst @@ -0,0 +1 @@ +Increases Windows required OS and API level to Windows 10. diff --git a/Misc/README b/Misc/README index 3dab768ba1a7a4..cbad9b72dc713c 100644 --- a/Misc/README +++ b/Misc/README @@ -17,7 +17,6 @@ python.man UNIX man page for the python interpreter python.pc.in Package configuration info template for pkg-config README The file you're reading now README.AIX Information about using Python on AIX -README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds svnmap.txt Map of old SVN revs and branches to hg changeset ids, diff --git a/Misc/README.coverity b/Misc/README.coverity deleted file mode 100644 index f5e1bf6f28d245..00000000000000 --- a/Misc/README.coverity +++ /dev/null @@ -1,22 +0,0 @@ - -Coverity has a static analysis tool (Prevent) which is similar to Klocwork. -They run their tool on the Python source code (SVN head) on a daily basis. -The results are available at: - - http://scan.coverity.com/ - -About 20 people have access to the analysis reports. Other -people can be added by request. - -Prevent was first run on the Python 2.5 source code in March 2006. -There were originally about 100 defects reported. Some of these -were false positives. Over 70 issues were uncovered. - -Each warning has a unique id and comments that can be made on it. -When checking in changes due to a warning, the unique id -as reported by the tool was added to the SVN commit message. - -False positives were annotated so that the comments can -be reviewed and reversed if the analysis was incorrect. - -Contact python-dev@python.org for more information. diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c deleted file mode 100644 index 90c72c7baa3f9e..00000000000000 --- a/Misc/coverity_model.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Coverity Scan model - * - * This is a modeling file for Coverity Scan. Modeling helps to avoid false - * positives. - * - * - A model file can't import any header files. - * - Therefore only some built-in primitives like int, char and void are - * available but not wchar_t, NULL etc. - * - Modeling doesn't need full structs and typedefs. Rudimentary structs - * and similar types are sufficient. - * - An uninitialized local pointer is not an error. It signifies that the - * variable could be either NULL or have some data. - * - * Coverity Scan doesn't pick up modifications automatically. The model file - * must be uploaded by an admin in the analysis settings of - * http://scan.coverity.com/projects/200 - */ - -/* dummy definitions, in most cases struct fields aren't required. */ - -#define NULL (void *)0 -#define assert(op) /* empty */ -typedef int sdigit; -typedef long Py_ssize_t; -typedef unsigned short wchar_t; -typedef struct {} PyObject; -typedef struct {} grammar; -typedef struct {} DIR; -typedef struct {} RFILE; - -/* Python/pythonrun.c - * resource leak false positive */ - -void Py_FatalError(const char *msg) { - __coverity_panic__(); -} - -/* Objects/longobject.c - * NEGATIVE_RETURNS false positive */ - -static PyObject *get_small_int(sdigit ival) -{ - /* Never returns NULL */ - PyObject *p; - assert(p != NULL); - return p; -} - -PyObject *PyLong_FromLong(long ival) -{ - PyObject *p; - int maybe; - - if ((ival >= -5) && (ival < 257 + 5)) { - p = get_small_int(ival); - assert(p != NULL); - return p; - } - if (maybe) - return p; - else - return NULL; -} - -PyObject *PyLong_FromLongLong(long long ival) -{ - return PyLong_FromLong((long)ival); -} - -PyObject *PyLong_FromSsize_t(Py_ssize_t ival) -{ - return PyLong_FromLong((long)ival); -} - -/* tainted sinks - * - * Coverity considers argv, environ, read() data etc as tainted. - */ - -PyObject *PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) -{ - __coverity_tainted_data_sink__(filename); - return NULL; -} - -/* Python/fileutils.c */ -wchar_t *Py_DecodeLocale(const char* arg, size_t *size) -{ - wchar_t *w; - __coverity_tainted_data_sink__(arg); - __coverity_tainted_data_sink__(size); - return w; -} - -/* Python/marshal.c */ - -static Py_ssize_t r_string(char *s, Py_ssize_t n, RFILE *p) -{ - __coverity_tainted_string_argument__(s); - return 0; -} - -static long r_long(RFILE *p) -{ - long l; - unsigned char buffer[4]; - - r_string((char *)buffer, 4, p); - __coverity_tainted_string_sanitize_content__(buffer); - l = (long)buffer; - return l; -} - -/* Coverity doesn't understand that fdopendir() may take ownership of fd. */ - -DIR *fdopendir(int fd) -{ - DIR *d; - if (d) { - __coverity_close__(fd); - } - return d; -} - -/* Modules/_datetime.c - * - * Coverity thinks that the input values for these function come from a - * tainted source PyDateTime_DATE_GET_* macros use bit shifting. - */ -static PyObject * -build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) -{ - PyObject *result; - - __coverity_tainted_data_sanitize__(y); - __coverity_tainted_data_sanitize__(m); - __coverity_tainted_data_sanitize__(d); - __coverity_tainted_data_sanitize__(hh); - __coverity_tainted_data_sanitize__(mm); - __coverity_tainted_data_sanitize__(ss); - __coverity_tainted_data_sanitize__(dstflag); - - return result; -} - -static int -ymd_to_ord(int year, int month, int day) -{ - int ord = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return ord; -} - -static int -normalize_date(int *year, int *month, int *day) -{ - __coverity_tainted_data_sanitize__(*year); - __coverity_tainted_data_sanitize__(*month); - __coverity_tainted_data_sanitize__(*day); - - return 0; -} - -static int -weekday(int year, int month, int day) -{ - int w = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return w; -} - diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 9aa398a80efa1b..52c0f883d383db 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 32373f0799bfeb..471b42badc8e8c 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -979,6 +979,30 @@ _codecs_register_error_impl(PyObject *module, const char *errors, Py_RETURN_NONE; } +/*[clinic input] +_codecs._unregister_error -> bool + errors: str + / + +Un-register the specified error handler for the error handling `errors'. + +Only custom error handlers can be un-registered. An exception is raised +if the error handling is a built-in one (e.g., 'strict'), or if an error +occurs. + +Otherwise, this returns True if a custom handler has been successfully +un-registered, and False if no custom handler for the specified error +handling exists. + +[clinic start generated code]*/ + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors) +/*[clinic end generated code: output=28c22be667465503 input=a63ab9e9ce1686d4]*/ +{ + return _PyCodec_UnregisterError(errors); +} + /*[clinic input] _codecs.lookup_error name: str @@ -1044,6 +1068,7 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_ENCODE_METHODDEF _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF + _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index fbfed59995c21e..aef04248c7e73c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2179,6 +2179,8 @@ typedef struct { PyObject *default_factory; } defdictobject; +static PyType_Spec defdict_spec; + PyDoc_STRVAR(defdict_missing_doc, "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\ if self.default_factory is None: raise KeyError((key,))\n\ @@ -2358,23 +2360,16 @@ defdict_or(PyObject* left, PyObject* right) { PyObject *self, *other; - // Find module state - PyTypeObject *tp = Py_TYPE(left); - PyObject *mod = PyType_GetModuleByDef(tp, &_collectionsmodule); - if (mod == NULL) { - PyErr_Clear(); - tp = Py_TYPE(right); - mod = PyType_GetModuleByDef(tp, &_collectionsmodule); + int ret = PyType_GetBaseByToken(Py_TYPE(left), &defdict_spec, NULL); + if (ret < 0) { + return NULL; } - assert(mod != NULL); - collections_state *state = get_module_state(mod); - - if (PyObject_TypeCheck(left, state->defdict_type)) { + if (ret) { self = left; other = right; } else { - assert(PyObject_TypeCheck(right, state->defdict_type)); + assert(PyType_GetBaseByToken(Py_TYPE(right), &defdict_spec, NULL) == 1); self = right; other = left; } @@ -2454,6 +2449,7 @@ passed to the dict constructor, including keyword arguments.\n\ #define DEFERRED_ADDRESS(ADDR) 0 static PyType_Slot defdict_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, defdict_dealloc}, {Py_tp_repr, defdict_repr}, {Py_nb_or, defdict_or}, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index c9ee5687c2b5d9..ece6b13c78851f 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -159,16 +159,32 @@ typedef chtype attr_t; /* No attr_t type is available */ #define _CURSES_PAIR_CONTENT_FUNC pair_content #endif /* _NCURSES_EXTENDED_COLOR_FUNCS */ +typedef struct _cursesmodule_state { + PyObject *error; // PyCursesError + PyTypeObject *window_type; // PyCursesWindow_Type +} _cursesmodule_state; + +// For now, we keep a global state variable to prepare for PEP 489. +static _cursesmodule_state curses_global_state; + +static inline _cursesmodule_state * +get_cursesmodule_state(PyObject *Py_UNUSED(module)) +{ + return &curses_global_state; +} + +static inline _cursesmodule_state * +get_cursesmodule_state_by_win(PyCursesWindowObject *Py_UNUSED(win)) +{ + return &curses_global_state; +} + /*[clinic input] module _curses class _curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=43265c372c2887d6]*/ -/* Definition of exception curses.error */ - -static PyObject *PyCursesError; - /* Tells whether setupterm() has been called to initialise terminfo. */ static int curses_setupterm_called = FALSE; @@ -180,53 +196,113 @@ static int curses_start_color_called = FALSE; static const char *curses_screen_encoding = NULL; -/* Utility Macros */ -#define PyCursesSetupTermCalled \ +/* Utility Checking Procedures */ + +/* + * Function to check that 'funcname' has been called by testing + * the 'called' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise, this returns 1. + * + * Since this function can be called in functions that do not + * have a direct access to the module's state, the exception + * type is directly taken from the global state for now. + */ +static inline int +_PyCursesCheckFunction(int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + PyErr_Format(curses_global_state.error, "must call %s() first", funcname); + return 0; +} + +/* + * Function to check that 'funcname' has been called by testing + * the 'called'' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise this returns 1. + * + * The exception type is obtained from the 'module' state. + */ +static inline int +_PyCursesStatefulCheckFunction(PyObject *module, int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "must call %s() first", funcname); + return 0; +} + +#define PyCursesStatefulSetupTermCalled(MODULE) \ do { \ - if (curses_setupterm_called != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call (at least) setupterm() first"); \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_setupterm_called, \ + "setupterm")) \ + { \ return 0; \ } \ } while (0) -#define PyCursesInitialised \ - do { \ - if (curses_initscr_called != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call initscr() first"); \ - return 0; \ - } \ +#define PyCursesStatefulInitialised(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_initscr_called, \ + "initscr")) \ + { \ + return 0; \ + } \ } while (0) -#define PyCursesInitialisedColor \ - do { \ - if (curses_start_color_called != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call start_color() first"); \ - return 0; \ - } \ +#define PyCursesStatefulInitialisedColor(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_start_color_called, \ + "start_color")) \ + { \ + return 0; \ + } \ } while (0) /* Utility Functions */ +static inline void +_PyCursesSetError(_cursesmodule_state *state, const char *funcname) +{ + if (funcname == NULL) { + PyErr_SetString(state->error, catchall_ERR); + } + else { + PyErr_Format(state->error, "%s() returned ERR", funcname); + } +} + /* * Check the return code from a curses function and return None - * or raise an exception as appropriate. These are exported using the - * capsule API. + * or raise an exception as appropriate. */ static PyObject * -PyCursesCheckERR(int code, const char *fname) +PyCursesCheckERR(PyObject *module, int code, const char *fname) { if (code != ERR) { Py_RETURN_NONE; } else { - if (fname == NULL) { - PyErr_SetString(PyCursesError, catchall_ERR); - } else { - PyErr_Format(PyCursesError, "%s() returned ERR", fname); - } + _cursesmodule_state *state = get_cursesmodule_state(module); + _PyCursesSetError(state, fname); + return NULL; + } +} + +static PyObject * +PyCursesCheckERR_ForWin(PyCursesWindowObject *win, int code, const char *fname) +{ + if (code != ERR) { + Py_RETURN_NONE; + } else { + _cursesmodule_state *state = get_cursesmodule_state_by_win(win); + _PyCursesSetError(state, fname); return NULL; } } @@ -555,20 +631,17 @@ class component_converter(CConverter): static int func_PyCursesSetupTermCalled(void) { - PyCursesSetupTermCalled; - return 1; + return _PyCursesCheckFunction(curses_setupterm_called, "setupterm"); } static int func_PyCursesInitialised(void) { - PyCursesInitialised; - return 1; + return _PyCursesCheckFunction(curses_initscr_called, "initscr"); } static int func_PyCursesInitialisedColor(void) { - PyCursesInitialisedColor; - return 1; + return _PyCursesCheckFunction(curses_start_color_called, "start_color"); } /***************************************************************************** @@ -590,7 +663,7 @@ PyTypeObject PyCursesWindow_Type; #define Window_NoArgNoReturnFunction(X) \ static PyObject *PyCursesWindow_ ## X \ (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { return PyCursesCheckERR(X(self->win), # X); } + { return PyCursesCheckERR_ForWin(self, X(self->win), # X); } #define Window_NoArgTrueFalseFunction(X) \ static PyObject * PyCursesWindow_ ## X \ @@ -625,7 +698,7 @@ PyTypeObject PyCursesWindow_Type; { \ TYPE arg1; \ if (!PyArg_ParseTuple(args,PARSESTR, &arg1)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1), # X); } + return PyCursesCheckERR_ForWin(self, X(self->win, arg1), # X); } #define Window_TwoArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ @@ -633,7 +706,7 @@ PyTypeObject PyCursesWindow_Type; { \ TYPE arg1, arg2; \ if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1, arg2), # X); } + return PyCursesCheckERR_ForWin(self, X(self->win, arg1, arg2), # X); } /* ------------- WINDOW routines --------------- */ @@ -807,7 +880,7 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, else { return NULL; } - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -887,7 +960,7 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -970,7 +1043,7 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -994,7 +1067,7 @@ _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr) if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; - return PyCursesCheckERR(wbkgd(self->win, bkgd | attr), "bkgd"); + return PyCursesCheckERR_ForWin(self, wbkgd(self->win, bkgd | attr), "bkgd"); } /*[clinic input] @@ -1010,7 +1083,7 @@ static PyObject * _curses_window_attroff_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=8a2fcd4df682fc64 input=786beedf06a7befe]*/ { - return PyCursesCheckERR(wattroff(self->win, (attr_t)attr), "attroff"); + return PyCursesCheckERR_ForWin(self, wattroff(self->win, (attr_t)attr), "attroff"); } /*[clinic input] @@ -1026,7 +1099,7 @@ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=7afea43b237fa870 input=5a88fba7b1524f32]*/ { - return PyCursesCheckERR(wattron(self->win, (attr_t)attr), "attron"); + return PyCursesCheckERR_ForWin(self, wattron(self->win, (attr_t)attr), "attron"); } /*[clinic input] @@ -1042,7 +1115,7 @@ static PyObject * _curses_window_attrset_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=84e379bff20c0433 input=42e400c0d0154ab5]*/ { - return PyCursesCheckERR(wattrset(self->win, (attr_t)attr), "attrset"); + return PyCursesCheckERR_ForWin(self, wattrset(self->win, (attr_t)attr), "attrset"); } /*[clinic input] @@ -1068,7 +1141,7 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, return NULL; wbkgdset(self->win, bkgd | attr); - return PyCursesCheckERR(0, "bkgdset"); + return PyCursesCheckERR_ForWin(self, 0, "bkgdset"); } /*[clinic input] @@ -1268,7 +1341,7 @@ PyCursesWindow_ChgAt(PyCursesWindowObject *self, PyObject *args) rtn = wchgat(self->win,num,attr,color,NULL); touchline(self->win,y,1); } - return PyCursesCheckERR(rtn, "chgat"); + return PyCursesCheckERR_ForWin(self, rtn, "chgat"); } #endif @@ -1292,10 +1365,10 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ { if (!group_right_1) { - return PyCursesCheckERR(wdelch(self->win), "wdelch"); + return PyCursesCheckERR_ForWin(self, wdelch(self->win), "wdelch"); } else { - return PyCursesCheckERR(py_mvwdelch(self->win, y, x), "mvwdelch"); + return PyCursesCheckERR_ForWin(self, py_mvwdelch(self->win, y, x), "mvwdelch"); } } @@ -1331,7 +1404,8 @@ _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, win = derwin(self->win,nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + _cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } @@ -1363,13 +1437,15 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, #ifdef py_is_pad if (py_is_pad(self->win)) { - return PyCursesCheckERR(pechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + pechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } else #endif - return PyCursesCheckERR(wechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + wechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } #ifdef NCURSES_MOUSE_VERSION @@ -1480,8 +1556,10 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, if (rtn == ERR) { /* getch() returns ERR in nodelay mode */ PyErr_CheckSignals(); - if (!PyErr_Occurred()) - PyErr_SetString(PyCursesError, "no input"); + if (!PyErr_Occurred()) { + _cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); + } return NULL; } else if (rtn <= 255) { #ifdef NCURSES_VERSION_MAJOR @@ -1539,7 +1617,8 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, return NULL; /* get_wch() returns ERR in nodelay mode */ - PyErr_SetString(PyCursesError, "no input"); + _cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); return NULL; } if (ct == KEY_CODE_YES) @@ -1663,10 +1742,10 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) { - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } } - return PyCursesCheckERR(whline(self->win, ch_ | (attr_t)attr, n), "hline"); + return PyCursesCheckERR_ForWin(self, whline(self->win, ch_ | (attr_t)attr, n), "hline"); } /*[clinic input] @@ -1713,7 +1792,7 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, rtn = mvwinsch(self->win, y, x, ch_ | (attr_t)attr); } - return PyCursesCheckERR(rtn, "insch"); + return PyCursesCheckERR_ForWin(self, rtn, "insch"); } /*[clinic input] @@ -1890,7 +1969,7 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -1975,7 +2054,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -2052,7 +2131,8 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + _cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "noutrefresh() called for a pad " "requires 6 arguments"); return NULL; @@ -2061,7 +2141,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) rtn = pnoutrefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "pnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "pnoutrefresh"); } if (group_right_1) { PyErr_SetString(PyExc_TypeError, @@ -2072,7 +2152,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) Py_BEGIN_ALLOW_THREADS rtn = wnoutrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "wnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "wnoutrefresh"); } /*[clinic input] @@ -2114,11 +2194,11 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, TRUE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overlay(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overlay"); + return PyCursesCheckERR_ForWin(self, rtn, "overlay"); } } @@ -2162,11 +2242,11 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, FALSE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overwrite(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overwrite"); + return PyCursesCheckERR_ForWin(self, rtn, "overwrite"); } } @@ -2195,7 +2275,7 @@ _curses_window_putwin(PyCursesWindowObject *self, PyObject *file) return PyErr_SetFromErrno(PyExc_OSError); if (_Py_set_inheritable(fileno(fp), 0, NULL) < 0) goto exit; - res = PyCursesCheckERR(putwin(self->win, fp), "putwin"); + res = PyCursesCheckERR_ForWin(self, putwin(self->win, fp), "putwin"); if (res == NULL) goto exit; fseek(fp, 0, 0); @@ -2234,7 +2314,7 @@ static PyObject * _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) /*[clinic end generated code: output=ea216e334f9ce1b4 input=152155e258a77a7a]*/ { - return PyCursesCheckERR(wredrawln(self->win,beg,num), "redrawln"); + return PyCursesCheckERR_ForWin(self, wredrawln(self->win,beg,num), "redrawln"); } /*[clinic input] @@ -2276,7 +2356,8 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + _cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "refresh() for a pad requires 6 arguments"); return NULL; } @@ -2284,7 +2365,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, rtn = prefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } #endif if (group_right_1) { @@ -2295,7 +2376,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, Py_BEGIN_ALLOW_THREADS rtn = wrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } /*[clinic input] @@ -2317,7 +2398,7 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, int bottom) /*[clinic end generated code: output=486ab5db218d2b1a input=1b517b986838bf0e]*/ { - return PyCursesCheckERR(wsetscrreg(self->win, top, bottom), "wsetscrreg"); + return PyCursesCheckERR_ForWin(self, wsetscrreg(self->win, top, bottom), "wsetscrreg"); } /*[clinic input] @@ -2358,7 +2439,8 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, win = subwin(self->win, nlines, ncols, begin_y, begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + _cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } @@ -2385,10 +2467,10 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ { if (!group_right_1) { - return PyCursesCheckERR(scroll(self->win), "scroll"); + return PyCursesCheckERR_ForWin(self, scroll(self->win), "scroll"); } else { - return PyCursesCheckERR(wscrl(self->win, lines), "scroll"); + return PyCursesCheckERR_ForWin(self, wscrl(self->win, lines), "scroll"); } } @@ -2414,10 +2496,10 @@ _curses_window_touchline_impl(PyCursesWindowObject *self, int start, /*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ { if (!group_right_1) { - return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); + return PyCursesCheckERR_ForWin(self, touchline(self->win, start, count), "touchline"); } else { - return PyCursesCheckERR(wtouchln(self->win, start, count, changed), "touchline"); + return PyCursesCheckERR_ForWin(self, wtouchln(self->win, start, count, changed), "touchline"); } } @@ -2457,9 +2539,9 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } - return PyCursesCheckERR(wvline(self->win, ch_ | (attr_t)attr, n), "vline"); + return PyCursesCheckERR_ForWin(self, wvline(self->win, ch_ | (attr_t)attr, n), "vline"); } static PyObject * @@ -2632,7 +2714,7 @@ PyTypeObject PyCursesWindow_Type = { PyCursesWindow_getsets, /* tp_getset */ }; -/* Function Prototype Macros - They are ugly but very, very useful. ;-) +/* Function Body Macros - They are ugly but very, very useful. ;-) X - function name TYPE - parameter Type @@ -2642,37 +2724,37 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgNoReturnFunctionBody(X) \ { \ - PyCursesInitialised; \ - return PyCursesCheckERR(X(), # X); } + PyCursesStatefulInitialised(module); \ + return PyCursesCheckERR(module, X(), # X); } #define NoArgOrFlagNoReturnFunctionBody(X, flag) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ if (flag) \ - return PyCursesCheckERR(X(), # X); \ + return PyCursesCheckERR(module, X(), # X); \ else \ - return PyCursesCheckERR(no ## X(), # X); \ + return PyCursesCheckERR(module, no ## X(), # X); \ } #define NoArgReturnIntFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ return PyLong_FromLong((long) X()); } #define NoArgReturnStringFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ return PyBytes_FromString(X()); } #define NoArgTrueFalseFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ return PyBool_FromLong(X()); } #define NoArgNoReturnVoidFunctionBody(X) \ { \ - PyCursesInitialised; \ + PyCursesStatefulInitialised(module); \ X(); \ Py_RETURN_NONE; } @@ -2770,12 +2852,13 @@ _curses_color_content_impl(PyObject *module, int color_number) { _CURSES_COLOR_VAL_TYPE r,g,b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_COLOR_CONTENT_FUNC(color_number, &r, &g, &b) == ERR) { - PyErr_Format(PyCursesError, "%s() returned ERR", - Py_STRINGIFY(_COLOR_CONTENT_FUNC)); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", + Py_STRINGIFY(_COLOR_CONTENT_FUNC)); return NULL; } @@ -2799,8 +2882,8 @@ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) /*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(COLOR_PAIR(pair_number)); } @@ -2826,10 +2909,10 @@ _curses_curs_set_impl(PyObject *module, int visibility) { int erg; - PyCursesInitialised; + PyCursesStatefulInitialised(module); erg = curs_set(visibility); - if (erg == ERR) return PyCursesCheckERR(erg, "curs_set"); + if (erg == ERR) return PyCursesCheckERR(module, erg, "curs_set"); return PyLong_FromLong((long) erg); } @@ -2878,9 +2961,9 @@ static PyObject * _curses_delay_output_impl(PyObject *module, int ms) /*[clinic end generated code: output=b6613a67f17fa4f4 input=5316457f5f59196c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(delay_output(ms), "delay_output"); + return PyCursesCheckERR(module, delay_output(ms), "delay_output"); } /*[clinic input] @@ -2934,7 +3017,7 @@ _curses_erasechar_impl(PyObject *module) { char ch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); ch = erasechar(); @@ -2984,7 +3067,7 @@ _curses_getsyx_impl(PyObject *module) int x = 0; int y = 0; - PyCursesInitialised; + PyCursesStatefulInitialised(module); getsyx(y, x); @@ -3009,11 +3092,12 @@ _curses_getmouse_impl(PyObject *module) int rtn; MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); rtn = getmouse( &event ); if (rtn == ERR) { - PyErr_SetString(PyCursesError, "getmouse() returned ERR"); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "getmouse() returned ERR"); return NULL; } return Py_BuildValue("(hiiik)", @@ -3044,14 +3128,14 @@ _curses_ungetmouse_impl(PyObject *module, short id, int x, int y, int z, { MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); event.id = id; event.x = x; event.y = y; event.z = z; event.bstate = bstate; - return PyCursesCheckERR(ungetmouse(&event), "ungetmouse"); + return PyCursesCheckERR(module, ungetmouse(&event), "ungetmouse"); } #endif @@ -3077,7 +3161,7 @@ _curses_getwin(PyObject *module, PyObject *file) WINDOW *win; PyObject *res = NULL; - PyCursesInitialised; + PyCursesStatefulInitialised(module); fp = tmpfile(); if (fp == NULL) @@ -3107,7 +3191,8 @@ _curses_getwin(PyObject *module, PyObject *file) fseek(fp, 0, 0); win = getwin(fp); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); goto error; } res = PyCursesWindow_New(win, NULL); @@ -3133,9 +3218,9 @@ static PyObject * _curses_halfdelay_impl(PyObject *module, unsigned char tenths) /*[clinic end generated code: output=e92cdf0ef33c0663 input=e42dce7259c15100]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(halfdelay(tenths), "halfdelay"); + return PyCursesCheckERR(module, halfdelay(tenths), "halfdelay"); } /*[clinic input] @@ -3186,7 +3271,7 @@ static PyObject * _curses_has_key_impl(PyObject *module, int key) /*[clinic end generated code: output=19ad48319414d0b1 input=78bd44acf1a4997c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(has_key(key)); } @@ -3217,10 +3302,11 @@ _curses_init_color_impl(PyObject *module, int color_number, short r, short g, short b) /*[clinic end generated code: output=d7ed71b2d818cdf2 input=ae2b8bea0f152c80]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); - return PyCursesCheckERR(_CURSES_INIT_COLOR_FUNC(color_number, r, g, b), + return PyCursesCheckERR(module, + _CURSES_INIT_COLOR_FUNC(color_number, r, g, b), Py_STRINGIFY(_CURSES_INIT_COLOR_FUNC)); } @@ -3245,8 +3331,8 @@ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) /*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_INIT_PAIR_FUNC(pair_number, fg, bg) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3255,7 +3341,8 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_INIT_PAIR_FUNC)); } return NULL; @@ -3286,7 +3373,8 @@ _curses_initscr_impl(PyObject *module) win = initscr(); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } @@ -3415,9 +3503,8 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) sys_stdout = PySys_GetObject("stdout"); if (sys_stdout == NULL || sys_stdout == Py_None) { - PyErr_SetString( - PyCursesError, - "lost sys.stdout"); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "lost sys.stdout"); return NULL; } @@ -3437,7 +3524,8 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) s = "setupterm: could not find terminfo database"; } - PyErr_SetString(PyCursesError,s); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, s); return NULL; } @@ -3487,7 +3575,7 @@ _curses_set_escdelay_impl(PyObject *module, int ms) return NULL; } - return PyCursesCheckERR(set_escdelay(ms), "set_escdelay"); + return PyCursesCheckERR(module, set_escdelay(ms), "set_escdelay"); } /*[clinic input] @@ -3526,7 +3614,7 @@ _curses_set_tabsize_impl(PyObject *module, int size) return NULL; } - return PyCursesCheckERR(set_tabsize(size), "set_tabsize"); + return PyCursesCheckERR(module, set_tabsize(size), "set_tabsize"); } #endif @@ -3542,9 +3630,9 @@ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(intrflush(NULL, flag), "intrflush"); + return PyCursesCheckERR(module, intrflush(NULL, flag), "intrflush"); } /*[clinic input] @@ -3575,7 +3663,7 @@ static PyObject * _curses_is_term_resized_impl(PyObject *module, int nlines, int ncols) /*[clinic end generated code: output=aafe04afe50f1288 input=ca9c0bd0fb8ab444]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(is_term_resized(nlines, ncols)); } @@ -3597,7 +3685,7 @@ _curses_keyname_impl(PyObject *module, int key) { const char *knp; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (key < 0) { PyErr_SetString(PyExc_ValueError, "invalid key number"); @@ -3655,9 +3743,9 @@ static PyObject * _curses_meta_impl(PyObject *module, int yes) /*[clinic end generated code: output=22f5abda46a605d8 input=cfe7da79f51d0e30]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(meta(stdscr, yes), "meta"); + return PyCursesCheckERR(module, meta(stdscr, yes), "meta"); } #ifdef NCURSES_MOUSE_VERSION @@ -3679,9 +3767,9 @@ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) /*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(mouseinterval(interval), "mouseinterval"); + return PyCursesCheckERR(module, mouseinterval(interval), "mouseinterval"); } /*[clinic input] @@ -3704,7 +3792,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) { mmask_t oldmask, availmask; - PyCursesInitialised; + PyCursesStatefulInitialised(module); availmask = mousemask((mmask_t)newmask, &oldmask); return Py_BuildValue("(kk)", (unsigned long)availmask, (unsigned long)oldmask); @@ -3725,7 +3813,7 @@ static int _curses_napms_impl(PyObject *module, int ms) /*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return napms(ms); } @@ -3749,12 +3837,13 @@ _curses_newpad_impl(PyObject *module, int nlines, int ncols) { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newpad(nlines, ncols); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } @@ -3789,11 +3878,12 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newwin(nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } @@ -3901,8 +3991,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) { _CURSES_COLOR_NUM_TYPE f, b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_PAIR_CONTENT_FUNC(pair_number, &f, &b) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3911,7 +4001,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_PAIR_CONTENT_FUNC)); } return NULL; @@ -3935,8 +4026,8 @@ static PyObject * _curses_pair_number_impl(PyObject *module, int attr) /*[clinic end generated code: output=85bce7d65c0aa3f4 input=d478548e33f5e61a]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(PAIR_NUMBER(attr)); } @@ -3956,7 +4047,7 @@ static PyObject * _curses_putp_impl(PyObject *module, const char *string) /*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ { - return PyCursesCheckERR(putp(string), "putp"); + return PyCursesCheckERR(module, putp(string), "putp"); } /*[clinic input] @@ -3976,7 +4067,7 @@ static PyObject * _curses_qiflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=9167e862f760ea30 input=6ec8b3e2b717ec40]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (flag) { qiflush(); @@ -4036,7 +4127,7 @@ update_lines_cols(PyObject *private_module) error: Py_XDECREF(o); - Py_DECREF(exposed_module); + Py_XDECREF(exposed_module); return 0; } @@ -4131,9 +4222,9 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols) { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resizeterm(nlines, ncols), "resizeterm"); + result = PyCursesCheckERR(module, resizeterm(nlines, ncols), "resizeterm"); if (!result) return NULL; if (!update_lines_cols(module)) { @@ -4170,9 +4261,9 @@ _curses_resize_term_impl(PyObject *module, int nlines, int ncols) { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resize_term(nlines, ncols), "resize_term"); + result = PyCursesCheckERR(module, resize_term(nlines, ncols), "resize_term"); if (!result) return NULL; if (!update_lines_cols(module)) { @@ -4213,7 +4304,7 @@ static PyObject * _curses_setsyx_impl(PyObject *module, int y, int x) /*[clinic end generated code: output=23dcf753511a2464 input=fa7f2b208e10a557]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); setsyx(y,x); @@ -4238,10 +4329,11 @@ static PyObject * _curses_start_color_impl(PyObject *module) /*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (start_color() == ERR) { - PyErr_SetString(PyCursesError, "start_color() returned ERR"); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "start_color() returned ERR"); return NULL; } @@ -4310,7 +4402,7 @@ static PyObject * _curses_tigetflag_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=8853c0e55542195b input=b0787af9e3e9a6ce]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetflag( (char *)capname ) ); } @@ -4332,7 +4424,7 @@ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetnum( (char *)capname ) ); } @@ -4354,7 +4446,7 @@ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); capname = tigetstr( (char *)capname ); if (capname == NULL || capname == (char*) -1) { @@ -4389,11 +4481,12 @@ _curses_tparm_impl(PyObject *module, const char *str, int i1, int i2, int i3, { char* result = NULL; - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); result = tparm((char *)str,i1,i2,i3,i4,i5,i6,i7,i8,i9); if (!result) { - PyErr_SetString(PyCursesError, "tparm() returned NULL"); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "tparm() returned NULL"); return NULL; } @@ -4417,9 +4510,9 @@ static PyObject * _curses_typeahead_impl(PyObject *module, int fd) /*[clinic end generated code: output=084bb649d7066583 input=f2968d8e1805051b]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(typeahead( fd ), "typeahead"); + return PyCursesCheckERR(module, typeahead( fd ), "typeahead"); } #endif @@ -4441,7 +4534,7 @@ _curses_unctrl(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; @@ -4464,12 +4557,12 @@ _curses_ungetch(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; - return PyCursesCheckERR(ungetch(ch_), "ungetch"); + return PyCursesCheckERR(module, ungetch(ch_), "ungetch"); } #ifdef HAVE_NCURSESW @@ -4535,11 +4628,11 @@ _curses_unget_wch(PyObject *module, PyObject *ch) { wchar_t wch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; - return PyCursesCheckERR(unget_wch(wch), "unget_wch"); + return PyCursesCheckERR(module, unget_wch(wch), "unget_wch"); } #endif @@ -4587,14 +4680,15 @@ _curses_use_default_colors_impl(PyObject *module) { int code; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); code = use_default_colors(); if (code != ERR) { Py_RETURN_NONE; } else { - PyErr_SetString(PyCursesError, "use_default_colors() returned ERR"); + _cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "use_default_colors() returned ERR"); return NULL; } } @@ -4785,6 +4879,7 @@ curses_destructor(PyObject *op) static int cursesmodule_exec(PyObject *module) { + _cursesmodule_state *state = get_cursesmodule_state(module); /* Initialize object type */ if (PyType_Ready(&PyCursesWindow_Type) < 0) { return -1; @@ -4792,6 +4887,7 @@ cursesmodule_exec(PyObject *module) if (PyModule_AddType(module, &PyCursesWindow_Type) < 0) { return -1; } + state->window_type = &PyCursesWindow_Type; /* Add some symbolic constants to the module */ PyObject *module_dict = PyModule_GetDict(module); @@ -4825,12 +4921,12 @@ cursesmodule_exec(PyObject *module) } /* For exception curses.error */ - PyCursesError = PyErr_NewException("_curses.error", NULL, NULL); - if (PyCursesError == NULL) { + state->error = PyErr_NewException("_curses.error", NULL, NULL); + if (state->error == NULL) { return -1; } - rc = PyDict_SetItemString(module_dict, "error", PyCursesError); - Py_DECREF(PyCursesError); + rc = PyDict_SetItemString(module_dict, "error", state->error); + Py_DECREF(state->error); if (rc < 0) { return -1; } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0d91bef2ff9bda..90527d2a3e0350 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5959,6 +5959,8 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) invalid_iso_midnight: PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + Py_DECREF(tzinfo); + Py_DECREF(dtstr_clean); return NULL; invalid_string_error: diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e99a96ab93392e..a33c9793b5ad17 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -122,6 +122,8 @@ get_module_state(PyObject *mod) } static struct PyModuleDef _decimal_module; +static PyType_Spec dec_spec; +static PyType_Spec context_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -134,10 +136,16 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = _PyType_GetModuleByDef2(Py_TYPE(left), Py_TYPE(right), - &_decimal_module); - assert(mod != NULL); - return get_module_state(mod); + PyTypeObject *base; + if (PyType_GetBaseByToken(Py_TYPE(left), &dec_spec, &base) != 1) { + assert(!PyErr_Occurred()); + PyType_GetBaseByToken(Py_TYPE(right), &dec_spec, &base); + } + assert(base != NULL); + void *state = _PyType_GetModuleState(base); + assert(state != NULL); + Py_DECREF(base); + return (decimal_state *)state; } @@ -183,6 +191,7 @@ typedef struct PyDecContextObject { PyObject *flags; int capitals; PyThreadState *tstate; + decimal_state *modstate; } PyDecContextObject; typedef struct { @@ -203,6 +212,15 @@ typedef struct { #define CTX(v) (&((PyDecContextObject *)v)->ctx) #define CtxCaps(v) (((PyDecContextObject *)v)->capitals) +static inline decimal_state * +get_module_state_from_ctx(PyObject *v) +{ + assert(PyType_GetBaseByToken(Py_TYPE(v), &context_spec, NULL) == 1); + decimal_state *state = ((PyDecContextObject *)v)->modstate; + assert(state != NULL); + return state; +} + Py_LOCAL_INLINE(PyObject *) incr_true(void) @@ -557,7 +575,7 @@ static int dec_addstatus(PyObject *context, uint32_t status) { mpd_context_t *ctx = CTX(context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); ctx->status |= status; if (status & (ctx->traps|MPD_Malloc_error)) { @@ -745,7 +763,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) { PyObject *res = Py_NotImplemented; - decimal_state *state = find_state_left_or_right(v, w); + decimal_state *state = get_module_state_by_def(Py_TYPE(v)); assert(PyDecSignalDict_Check(state, v)); if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { @@ -852,7 +870,7 @@ static PyObject * context_getround(PyObject *self, void *Py_UNUSED(closure)) { int i = mpd_getround(CTX(self)); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); return Py_NewRef(state->round_map[i]); } @@ -1011,7 +1029,7 @@ context_setround(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) mpd_context_t *ctx; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); x = getround(state, value); if (x == -1) { return -1; @@ -1070,7 +1088,7 @@ context_settraps_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { return -1; @@ -1090,7 +1108,7 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1135,7 +1153,7 @@ context_setstatus_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { @@ -1156,7 +1174,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1386,6 +1404,7 @@ context_new(PyTypeObject *type, CtxCaps(self) = 1; self->tstate = NULL; + self->modstate = state; if (type == state->PyDecContext_Type) { PyObject_GC_Track(self); @@ -1463,7 +1482,7 @@ context_repr(PyDecContextObject *self) int n, mem; #ifdef Py_DEBUG - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx((PyObject *)self); assert(PyDecContext_Check(state, self)); #endif ctx = CTX(self); @@ -1554,7 +1573,7 @@ context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *copy; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; @@ -1574,7 +1593,7 @@ context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); ctx = CTX(self); @@ -2015,11 +2034,10 @@ static PyType_Spec ctxmanager_spec = { /******************************************************************************/ static PyObject * -PyDecType_New(PyTypeObject *type) +PyDecType_New(decimal_state *state, PyTypeObject *type) { PyDecObject *dec; - decimal_state *state = get_module_state_by_def(type); if (type == state->PyDec_Type) { dec = PyObject_GC_New(PyDecObject, state->PyDec_Type); } @@ -2045,7 +2063,7 @@ PyDecType_New(PyTypeObject *type) assert(PyObject_GC_IsTracked((PyObject *)dec)); return (PyObject *)dec; } -#define dec_alloc(st) PyDecType_New((st)->PyDec_Type) +#define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) static int dec_traverse(PyObject *dec, visitproc visit, void *arg) @@ -2148,7 +2166,8 @@ PyDecType_FromCString(PyTypeObject *type, const char *s, PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2172,7 +2191,8 @@ PyDecType_FromCStringExact(PyTypeObject *type, const char *s, uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2259,7 +2279,8 @@ PyDecType_FromSsize(PyTypeObject *type, mpd_ssize_t v, PyObject *context) PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2280,7 +2301,8 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2298,13 +2320,13 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) /* Convert from a PyLongObject. The context is not modified; flags set during conversion are accumulated in the status parameter. */ static PyObject * -dec_from_long(PyTypeObject *type, PyObject *v, +dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { PyObject *dec; PyLongObject *l = (PyLongObject *)v; - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2349,7 +2371,8 @@ PyDecType_FromLong(PyTypeObject *type, PyObject *v, PyObject *context) return NULL; } - dec = dec_from_long(type, v, CTX(context), &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2378,7 +2401,8 @@ PyDecType_FromLongExact(PyTypeObject *type, PyObject *v, } mpd_maxcontext(&maxctx); - dec = dec_from_long(type, v, &maxctx, &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, &maxctx, &status); if (dec == NULL) { return NULL; } @@ -2410,7 +2434,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, mpd_t *d1, *d2; uint32_t status = 0; mpd_context_t maxctx; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); #ifdef Py_DEBUG assert(PyType_IsSubtype(type, state->PyDec_Type)); @@ -2431,7 +2455,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, sign = (copysign(1.0, x) == 1.0) ? 0 : 1; if (isnan(x) || isinf(x)) { - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2548,12 +2572,12 @@ PyDecType_FromDecimalExact(PyTypeObject *type, PyObject *v, PyObject *context) PyObject *dec; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (type == state->PyDec_Type && PyDec_CheckExact(state, v)) { return Py_NewRef(v); } - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2837,7 +2861,7 @@ dec_from_float(PyObject *type, PyObject *pyfloat) static PyObject * ctx_from_float(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); return PyDec_FromFloat(state, v, context); } @@ -2848,7 +2872,7 @@ dec_apply(PyObject *v, PyObject *context) PyObject *result; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -2875,7 +2899,7 @@ dec_apply(PyObject *v, PyObject *context) static PyObject * PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDecType_FromSsizeExact(type, 0, context); } @@ -2910,7 +2934,7 @@ PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) static PyObject * PyDec_FromObject(PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDec_FromSsize(state, 0, context); } @@ -2997,7 +3021,7 @@ ctx_create_decimal(PyObject *context, PyObject *args) Py_LOCAL_INLINE(int) convert_op(int type_err, PyObject **conv, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, v)) { *conv = Py_NewRef(v); return 1; @@ -3100,7 +3124,7 @@ multiply_by_denominator(PyObject *v, PyObject *r, PyObject *context) if (tmp == NULL) { return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); denom = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); if (denom == NULL) { @@ -3155,7 +3179,7 @@ numerator_as_decimal(PyObject *r, PyObject *context) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); num = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); return num; @@ -3174,7 +3198,7 @@ convert_op_cmp(PyObject **vcmp, PyObject **wcmp, PyObject *v, PyObject *w, *vcmp = v; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, w)) { *wcmp = Py_NewRef(w); } @@ -4414,12 +4438,11 @@ dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) return Py_NewRef(self); } -static PyObject * -dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +static inline PyObject * +_dec_mpd_radix(decimal_state *state) { PyObject *result; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -4429,6 +4452,13 @@ dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +static PyObject * +dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +{ + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + return _dec_mpd_radix(state); +} + static PyObject * dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) { @@ -5041,6 +5071,7 @@ static PyMethodDef dec_methods [] = }; static PyType_Slot dec_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, dec_traverse}, @@ -5130,7 +5161,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ \ CONVERT_OP_RAISE(&a, v, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5162,7 +5193,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5198,7 +5229,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5227,7 +5258,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ } \ \ CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); \ + decimal_state *state = get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5293,7 +5324,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { Py_DECREF(a); @@ -5348,7 +5379,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5383,7 +5414,8 @@ DecCtx_TernaryFunc(mpd_qfma) static PyObject * ctx_mpd_radix(PyObject *context, PyObject *dummy) { - return dec_mpd_radix(context, dummy); + decimal_state *state = get_module_state_from_ctx(context); + return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ @@ -5400,7 +5432,7 @@ DecCtx_BoolFunc_NO_CTX(mpd_iszero) static PyObject * ctx_iscanonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5426,7 +5458,7 @@ PyDecContext_Apply(PyObject *context, PyObject *v) static PyObject * ctx_canonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5443,7 +5475,7 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5476,7 +5508,7 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5573,7 +5605,7 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5728,6 +5760,7 @@ static PyMethodDef context_methods [] = }; static PyType_Slot context_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, context_dealloc}, {Py_tp_traverse, context_traverse}, {Py_tp_clear, context_clear}, diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 2b3bd7c3de1176..31cf7bcc09782c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -26,7 +26,7 @@ typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; PyTypeObject *placeholder_type; - PyObject *placeholder; + PyObject *placeholder; // strong reference (singleton) PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -76,13 +76,12 @@ static PyMethodDef placeholder_methods[] = { }; static void -placeholder_dealloc(PyObject* placeholder) +placeholder_dealloc(PyObject* self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref Placeholder out of existence. Instead, - * since Placeholder is an immortal object, re-set the reference count. - */ - _Py_SetImmortal(placeholder); + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); } static PyObject * @@ -93,10 +92,26 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder != NULL) { + return Py_NewRef(state->placeholder); + } + + PyObject *placeholder = PyType_GenericNew(type, NULL, NULL); + if (placeholder == NULL) { + return NULL; + } + if (state->placeholder == NULL) { - state->placeholder = PyType_GenericNew(type, NULL, NULL); + state->placeholder = Py_NewRef(placeholder); } - return state->placeholder; + return placeholder; +} + +static int +placeholder_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; } static PyType_Slot placeholder_type_slots[] = { @@ -105,13 +120,14 @@ static PyType_Slot placeholder_type_slots[] = { {Py_tp_doc, (void *)placeholder_doc}, {Py_tp_methods, placeholder_methods}, {Py_tp_new, placeholder_new}, + {Py_tp_traverse, placeholder_traverse}, {0, 0} }; static PyType_Spec placeholder_type_spec = { .name = "functools._PlaceholderType", .basicsize = sizeof(placeholderobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, .slots = placeholder_type_slots }; @@ -1717,13 +1733,17 @@ _functools_exec(PyObject *module) if (PyModule_AddType(module, state->placeholder_type) < 0) { return -1; } - state->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); - if (state->placeholder == NULL) { + + PyObject *placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (placeholder == NULL) { return -1; } - if (PyModule_AddObject(module, "Placeholder", state->placeholder) < 0) { + if (PyModule_AddObjectRef(module, "Placeholder", placeholder) < 0) { + Py_DECREF(placeholder); return -1; } + Py_DECREF(placeholder); + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1769,6 +1789,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); Py_VISIT(state->placeholder_type); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1781,6 +1802,7 @@ _functools_clear(PyObject *module) _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); Py_CLEAR(state->placeholder_type); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 18affdd4875f3b..b2bd9545c1b130 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2146,7 +2146,7 @@ save_long(PicklerObject *self, PyObject *obj) if (self->proto >= 2) { /* Linear-time pickling. */ - uint64_t nbits; + int64_t nbits; size_t nbytes; unsigned char *pdata; char header[5]; @@ -2161,8 +2161,8 @@ save_long(PicklerObject *self, PyObject *obj) return 0; } nbits = _PyLong_NumBits(obj); - if (nbits == (uint64_t)-1 && PyErr_Occurred()) - goto error; + assert(nbits >= 0); + assert(!PyErr_Occurred()); /* How many bytes do we need? There are nbits >> 3 full * bytes of data, and nbits & 7 leftover bits. If there * are any leftover bits, then we clearly need another diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 3835a3072d96c6..ad66df47349db0 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -295,7 +295,7 @@ random_seed(RandomObject *self, PyObject *arg) int result = -1; /* guilty until proved innocent */ PyObject *n = NULL; uint32_t *key = NULL; - uint64_t bits; + int64_t bits; size_t keyused; int res; @@ -335,8 +335,8 @@ random_seed(RandomObject *self, PyObject *arg) /* Now split n into 32-bit chunks, from the right. */ bits = _PyLong_NumBits(n); - if (bits == (uint64_t)-1 && PyErr_Occurred()) - goto Done; + assert(bits >= 0); + assert(!PyErr_Occurred()); /* Figure out how many 32-bit chunks this gives us. */ keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 1f5f0215980971..f2d3b331226a7a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2251,6 +2251,17 @@ PySSL_dealloc(PySSLSocket *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); if (self->ssl) { + // If we free the SSL socket object without having called SSL_shutdown, + // OpenSSL will invalidate the linked SSL session object. While this + // behavior is strictly RFC-compliant, it makes session reuse less + // likely and it would also break compatibility with older stdlib + // versions (which used an ugly workaround of duplicating the + // SSL_SESSION object). + // Therefore, we ensure the socket is marked as shutdown in any case. + // + // See elaborate explanation at + // https://github.com/python/cpython/pull/123249#discussion_r1766164530 + SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } Py_XDECREF(self->Socket); @@ -2795,48 +2806,6 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) #endif } -static SSL_SESSION* -_ssl_session_dup(SSL_SESSION *session) { - SSL_SESSION *newsession = NULL; - int slen; - unsigned char *senc = NULL, *p; - const unsigned char *const_p; - - if (session == NULL) { - PyErr_SetString(PyExc_ValueError, "Invalid session"); - goto error; - } - - /* get length */ - slen = i2d_SSL_SESSION(session, NULL); - if (slen == 0 || slen > 0xFF00) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - if ((senc = PyMem_Malloc(slen)) == NULL) { - PyErr_NoMemory(); - goto error; - } - p = senc; - if (!i2d_SSL_SESSION(session, &p)) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - const_p = senc; - newsession = d2i_SSL_SESSION(NULL, &const_p, slen); - if (newsession == NULL) { - PyErr_SetString(PyExc_ValueError, "d2i() failed"); - goto error; - } - PyMem_Free(senc); - return newsession; - error: - if (senc != NULL) { - PyMem_Free(senc); - } - return NULL; -} - static PyObject * PySSL_get_session(PySSLSocket *self, void *closure) { /* get_session can return sessions from a server-side connection, @@ -2844,15 +2813,6 @@ PySSL_get_session(PySSLSocket *self, void *closure) { PySSLSession *pysess; SSL_SESSION *session; - /* duplicate session as workaround for session bug in OpenSSL 1.1.0, - * https://github.com/openssl/openssl/issues/1550 */ - session = SSL_get0_session(self->ssl); /* borrowed reference */ - if (session == NULL) { - Py_RETURN_NONE; - } - if ((session = _ssl_session_dup(session)) == NULL) { - return NULL; - } session = SSL_get1_session(self->ssl); if (session == NULL) { Py_RETURN_NONE; @@ -2871,11 +2831,8 @@ PySSL_get_session(PySSLSocket *self, void *closure) { } static int PySSL_set_session(PySSLSocket *self, PyObject *value, - void *closure) - { + void *closure) { PySSLSession *pysess; - SSL_SESSION *session; - int result; if (!Py_IS_TYPE(value, get_state_sock(self)->PySSLSession_Type)) { PyErr_SetString(PyExc_TypeError, "Value is not a SSLSession."); @@ -2898,14 +2855,7 @@ static int PySSL_set_session(PySSLSocket *self, PyObject *value, "Cannot set session after handshake."); return -1; } - /* duplicate session */ - if ((session = _ssl_session_dup(pysess->session)) == NULL) { - return -1; - } - result = SSL_set_session(self->ssl, session); - /* free duplicate, SSL_set_session() bumps ref count */ - SSL_SESSION_free(session); - if (result == 0) { + if (SSL_set_session(self->ssl, pysess->session) == 0) { _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); return -1; } diff --git a/Modules/_testcapi/codec.c b/Modules/_testcapi/codec.c index d13f51e20331a1..e27e64e066c458 100644 --- a/Modules/_testcapi/codec.c +++ b/Modules/_testcapi/codec.c @@ -1,15 +1,219 @@ #include "parts.h" #include "util.h" +// === Codecs registration and un-registration ================================ + +static PyObject * +codec_register(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Register(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_unregister(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Unregister(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_known_encoding(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_KnownEncoding(encoding) ? Py_True : Py_False; +} + +// === Codecs encoding and decoding interfaces ================================ + +static PyObject * +codec_encode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Encode(input, encoding, errors); +} + +static PyObject * +codec_decode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Decode(input, encoding, errors); +} + +static PyObject * +codec_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Encoder(encoding); +} + +static PyObject * +codec_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Decoder(encoding); +} + +static PyObject * +codec_incremental_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incremental_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +codec_stream_reader(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamReader(encoding, stream, errors); +} + +static PyObject * +codec_stream_writer(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamWriter(encoding, stream, errors); +} + +// === Codecs errors handlers ================================================= + +static PyObject * +codec_register_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *encoding; // must not be NULL + PyObject *error; + if (!PyArg_ParseTuple(args, "sO", &encoding, &error)) { + return NULL; + } + if (PyCodec_RegisterError(encoding, error) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_lookup_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_LookupError(encoding); +} + +static PyObject * +codec_strict_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_StrictErrors(exc); +} + +static PyObject * +codec_ignore_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_IgnoreErrors(exc); +} + +static PyObject * +codec_replace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_ReplaceErrors(exc); +} + +static PyObject * +codec_xmlcharrefreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_XMLCharRefReplaceErrors(exc); +} + +static PyObject * +codec_backslashreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_BackslashReplaceErrors(exc); +} static PyMethodDef test_methods[] = { - {NULL}, + /* codecs registration */ + {"codec_register", codec_register, METH_O}, + {"codec_unregister", codec_unregister, METH_O}, + {"codec_known_encoding", codec_known_encoding, METH_VARARGS}, + /* encoding and decoding interface */ + {"codec_encode", codec_encode, METH_VARARGS}, + {"codec_decode", codec_decode, METH_VARARGS}, + {"codec_encoder", codec_encoder, METH_VARARGS}, + {"codec_decoder", codec_decoder, METH_VARARGS}, + {"codec_incremental_encoder", codec_incremental_encoder, METH_VARARGS}, + {"codec_incremental_decoder", codec_incremental_decoder, METH_VARARGS}, + {"codec_stream_reader", codec_stream_reader, METH_VARARGS}, + {"codec_stream_writer", codec_stream_writer, METH_VARARGS}, + /* error handling */ + {"codec_register_error", codec_register_error, METH_VARARGS}, + {"codec_lookup_error", codec_lookup_error, METH_VARARGS}, + {"codec_strict_errors", codec_strict_errors, METH_O}, + {"codec_ignore_errors", codec_ignore_errors, METH_O}, + {"codec_replace_errors", codec_replace_errors, METH_O}, + {"codec_xmlcharrefreplace_errors", codec_xmlcharrefreplace_errors, METH_O}, + {"codec_backslashreplace_errors", codec_backslashreplace_errors, METH_O}, + // PyCodec_NameReplaceErrors() is tested in _testlimitedcapi/codec.c + {NULL, NULL, 0, NULL}, }; int _PyTestCapi_Init_Codec(PyObject *m) { - if (PyModule_AddFunctions(m, test_methods) < 0){ + if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; } diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index e80d898118daa5..307797f98f12ae 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -181,19 +181,6 @@ dict_popstring_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_PopString(dict, key, NULL)); } -static PyObject * -dict_version(PyObject *self, PyObject *dict) -{ - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, "expected dict"); - return NULL; - } -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag); -_Py_COMP_DIAG_POP -} - static PyMethodDef test_methods[] = { {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_getitemref", dict_getitemref, METH_VARARGS}, @@ -204,7 +191,6 @@ static PyMethodDef test_methods[] = { {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, - {"dict_version", dict_version, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/util.h b/Modules/_testcapi/util.h index f26d7656a10138..042e522542eddb 100644 --- a/Modules/_testcapi/util.h +++ b/Modules/_testcapi/util.h @@ -31,3 +31,13 @@ static const char uninitialized[] = "uninitialized"; #define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) /* Marker to check that integer value was set. */ #define UNINITIALIZED_INT (63256717) +/* + * Marker to indicate that a NULL parameter would not be allowed + * at runtime but that the test interface will check that it is + * indeed the case. + * + * Use this macro only if passing NULL to the C API would raise + * a catchable exception (and not a fatal exception that would + * crash the interpreter). + */ + #define NULL_WOULD_RAISE(NAME) NAME diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5966eb674cf4e5..72b9792d7005ff 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1909,25 +1909,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -dict_get_version(PyObject *self, PyObject *args) -{ - PyDictObject *dict; - uint64_t version; - - if (!PyArg_ParseTuple(args, "O!", &PyDict_Type, &dict)) - return NULL; - - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS - version = dict->ma_version_tag; - _Py_COMP_DIAG_POP - - static_assert(sizeof(unsigned long long) >= sizeof(version), - "version is larger than unsigned long long"); - return PyLong_FromUnsignedLongLong((unsigned long long)version); -} - static PyObject * raise_SIGINT_then_send_None(PyObject *self, PyObject *args) @@ -3407,7 +3388,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, #ifdef W_STOPCODE diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index e74cbfe19871bf..ba83a23117b2a5 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -38,6 +38,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Codec(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Complex(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/codec.c b/Modules/_testlimitedcapi/codec.c new file mode 100644 index 00000000000000..fdc18eedc2d288 --- /dev/null +++ b/Modules/_testlimitedcapi/codec.c @@ -0,0 +1,29 @@ +#include "pyconfig.h" // Py_GIL_DISABLED + +// Need limited C API version 3.5 for PyCodec_NameReplaceErrors() +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" + +static PyObject * +codec_namereplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_NameReplaceErrors(exc); +} + +static PyMethodDef test_methods[] = { + {"codec_namereplace_errors", codec_namereplace_errors, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Codec(PyObject *module) +{ + if (PyModule_AddFunctions(module, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 12b890853803f4..4107b150c5b4e0 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,6 +25,7 @@ int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Codec(PyObject *module); int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); int _PyTestLimitedCAPI_Init_Eval(PyObject *module); diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab350..01855aec5e123e 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -2683,6 +2683,56 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_codecs__unregister_error__doc__, +"_unregister_error($module, errors, /)\n" +"--\n" +"\n" +"Un-register the specified error handler for the error handling `errors\'.\n" +"\n" +"Only custom error handlers can be un-registered. An exception is raised\n" +"if the error handling is a built-in one (e.g., \'strict\'), or if an error\n" +"occurs.\n" +"\n" +"Otherwise, this returns True if a custom handler has been successfully\n" +"un-registered, and False if no custom handler for the specified error\n" +"handling exists."); + +#define _CODECS__UNREGISTER_ERROR_METHODDEF \ + {"_unregister_error", (PyCFunction)_codecs__unregister_error, METH_O, _codecs__unregister_error__doc__}, + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors); + +static PyObject * +_codecs__unregister_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *errors; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_unregister_error", "argument", "str", arg); + goto exit; + } + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + _return_value = _codecs__unregister_error_impl(module, errors); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_codecs_lookup_error__doc__, "lookup_error($module, name, /)\n" "--\n" @@ -2746,4 +2796,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3013d4709d96ffe input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index baf2dc439b8959..058f57770755aa 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1657,7 +1657,7 @@ math_isqrt(PyObject *module, PyObject *n) /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ { int a_too_large, c_bit_length; - uint64_t c, d; + int64_t c, d; uint64_t m; uint32_t u; PyObject *a = NULL, *b; @@ -1680,14 +1680,13 @@ math_isqrt(PyObject *module, PyObject *n) /* c = (n.bit_length() - 1) // 2 */ c = _PyLong_NumBits(n); - if (c == (uint64_t)(-1)) { - goto error; - } - c = (c - 1U) / 2U; + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a fast, almost branch-free algorithm. */ - if (c <= 31U) { + if (c <= 31) { int shift = 31 - (int)c; m = (uint64_t)PyLong_AsUnsignedLongLong(n); Py_DECREF(n); @@ -1704,13 +1703,13 @@ math_isqrt(PyObject *module, PyObject *n) /* From n >= 2**64 it follows that c.bit_length() >= 6. */ c_bit_length = 6; - while ((c >> c_bit_length) > 0U) { + while ((c >> c_bit_length) > 0) { ++c_bit_length; } /* Initialise d and a. */ d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2U*c - 62U); + b = _PyLong_Rshift(n, 2*c - 62); if (b == NULL) { goto error; } @@ -1727,12 +1726,12 @@ math_isqrt(PyObject *module, PyObject *n) for (int s = c_bit_length - 6; s >= 0; --s) { PyObject *q; - uint64_t e = d; + int64_t e = d; d = c >> s; /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2U*c - d - e + 1U); + q = _PyLong_Rshift(n, 2*c - d - e + 1); if (q == NULL) { goto error; } @@ -1742,7 +1741,7 @@ math_isqrt(PyObject *module, PyObject *n) } /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e)); + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); if (a == NULL) { Py_DECREF(q); goto error; @@ -2202,8 +2201,8 @@ loghelper(PyObject* arg, double (*func)(double)) to compute the log anyway. Clear the exception and continue. */ PyErr_Clear(); x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; + assert(e >= 0); + assert(!PyErr_Occurred()); /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ result = func(x) + func(2.0) * e; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ee59fb73ac1e31..9720c201a184a8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -813,7 +813,12 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +// Some platforms only support a limited range of years. +// +// Android works with negative years on the emulator, but fails on some +// physical devices (#123017). +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ + || defined(__VXWORKS__) || defined(__ANDROID__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); diff --git a/Objects/capsule.c b/Objects/capsule.c index 555979dab2b789..28965e0f21b7a0 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -317,10 +317,14 @@ static int capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) { // Capsule object is only tracked by the GC - // if _PyCapsule_SetTraverse() is called - assert(capsule->traverse_func != NULL); + // if _PyCapsule_SetTraverse() is called, but + // this can still be manually triggered by gc.get_referents() + + if (capsule->traverse_func != NULL) { + return capsule->traverse_func((PyObject*)capsule, visit, arg); + } - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return 0; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f38ab1b2865e99..adfd91d1e4d63b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -877,7 +877,7 @@ new_dict(PyInterpreterState *interp, mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(interp); + mp->_ma_watcher_tag = 0; ASSERT_CONSISTENT(mp); return (PyObject *)mp; } @@ -1488,7 +1488,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb *value_addr = value; if (value != NULL) { assert(ix >= 0); - Py_INCREF(value); + _Py_NewRefWithLock(value); } Py_END_CRITICAL_SECTION(); return ix; @@ -1678,8 +1678,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, } } - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); mp->ma_keys->dk_version = 0; Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); @@ -1698,7 +1697,6 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, STORE_VALUE(ep, value); STORE_HASH(ep, hash); } - mp->ma_version_tag = new_version; STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); assert(mp->ma_keys->dk_usable >= 0); @@ -1744,16 +1742,14 @@ insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, MAINTAIN_TRACKING(mp, key, value); PyObject *old_value = mp->ma_values->values[ix]; if (old_value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); STORE_USED(mp, mp->ma_used + 1); - mp->ma_version_tag = new_version; } else { - uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); - mp->ma_version_tag = new_version; // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_DECREF(old_value); @@ -1815,8 +1811,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_MODIFIED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); assert(!_PyDict_HasSplitTable(mp)); if (DK_IS_UNICODE(mp->ma_keys)) { @@ -1827,7 +1822,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; STORE_VALUE(ep, value); } - mp->ma_version_tag = new_version; } Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ ASSERT_CONSISTENT(mp); @@ -1857,8 +1851,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ assert(mp->ma_values == NULL); @@ -1879,7 +1872,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, STORE_VALUE(ep, value); } STORE_USED(mp, mp->ma_used + 1); - mp->ma_version_tag = new_version; newkeys->dk_usable--; newkeys->dk_nentries++; // We store the keys last so no one can see them in a partially inconsistent @@ -2612,7 +2604,7 @@ delete_index_from_values(PyDictValues *values, Py_ssize_t ix) static void delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, - PyObject *old_value, uint64_t new_version) + PyObject *old_value) { PyObject *old_key; @@ -2622,7 +2614,6 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, assert(hashpos >= 0); STORE_USED(mp, mp->ma_used - 1); - mp->ma_version_tag = new_version; if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); STORE_SPLIT_VALUE(mp, ix, NULL); @@ -2692,9 +2683,8 @@ delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) } PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); return 0; } @@ -2740,9 +2730,8 @@ delitemif_lock_held(PyObject *op, PyObject *key, if (res > 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); return 1; } else { return 0; @@ -2786,11 +2775,9 @@ clear_lock_held(PyObject *op) } /* Empty the dict... */ PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); // We don't inc ref empty keys because they're immortal ensure_shared_on_resize(mp); - mp->ma_version_tag = new_version; STORE_USED(mp, 0); if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); @@ -2950,9 +2937,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, assert(old_value != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, Py_NewRef(old_value)); ASSERT_CONSISTENT(mp); if (result) { @@ -3717,8 +3703,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) ) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) return -1; @@ -3727,7 +3712,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; STORE_USED(mp, other->ma_used); - mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { @@ -3930,13 +3914,13 @@ dict_copy_impl(PyDictObject *self) } /* Copies the values, but does not change the reference - * counts of the objects in the array. */ + * counts of the objects in the array. + * Return NULL, but does *not* set an exception on failure */ static PyDictValues * copy_values(PyDictValues *values) { PyDictValues *newvalues = new_values(values->capacity); if (newvalues == NULL) { - PyErr_NoMemory(); return NULL; } newvalues->size = values->size; @@ -3982,7 +3966,7 @@ copy_lock_held(PyObject *o) split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); + split_copy->_ma_watcher_tag = 0; dictkeys_incref(mp->ma_keys); if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); @@ -4430,7 +4414,6 @@ dict_popitem_impl(PyDictObject *self) { Py_ssize_t i, j; PyObject *res; - uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); ASSERT_DICT_LOCKED(self); @@ -4473,8 +4456,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -4489,8 +4471,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -4508,7 +4489,6 @@ dict_popitem_impl(PyDictObject *self) /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ STORE_KEYS_NENTRIES(self->ma_keys, i); STORE_USED(self, self->ma_used - 1); - self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; } @@ -4759,8 +4739,7 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION( - _PyInterpreterState_GET()); + d->_ma_watcher_tag = 0; dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -7015,7 +6994,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr // Still no dict, we can read from the values assert(values->valid); value = values->values[ix]; - *attr = Py_XNewRef(value); + *attr = _Py_XNewRefWithLock(value); success = true; } @@ -7035,7 +7014,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); - *attr = Py_XNewRef(value); + *attr = _Py_XNewRefWithLock(value); success = true; } else { // Caller needs to lookup from the dictionary @@ -7216,6 +7195,13 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) PyDictValues *values = copy_values(mp->ma_values); if (values == NULL) { + /* Out of memory. Clear the dict */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = mp->ma_keys; + set_keys(mp, Py_EMPTY_KEYS); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + STORE_USED(mp, 0); + PyErr_NoMemory(); return -1; } mp->ma_values = values; @@ -7377,7 +7363,7 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag |= (1LL << watcher_id); return 0; } @@ -7392,7 +7378,7 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag &= ~(1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag &= ~(1LL << watcher_id); return 0; } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index fda62f159c1540..b3910855165494 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3387,8 +3387,9 @@ _PyErr_NoMemory(PyThreadState *tstate) } static void -MemoryError_dealloc(PyBaseExceptionObject *self) +MemoryError_dealloc(PyObject *obj) { + PyBaseExceptionObject *self = (PyBaseExceptionObject *)obj; _PyObject_GC_UNTRACK(self); BaseException_clear(self); @@ -3447,7 +3448,7 @@ PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, + 0, MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, diff --git a/Objects/floatobject.c b/Objects/floatobject.c index dc3d8a3e5d0f4b..a48a210adee3b9 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -406,19 +406,16 @@ float_richcompare(PyObject *v, PyObject *w, int op) } /* The signs are the same. */ /* Convert w to a double if it fits. In particular, 0 fits. */ - uint64_t nbits64 = _PyLong_NumBits(w); - if (nbits64 > (unsigned int)DBL_MAX_EXP) { + int64_t nbits64 = _PyLong_NumBits(w); + assert(nbits64 >= 0); + assert(!PyErr_Occurred()); + if (nbits64 > DBL_MAX_EXP) { /* This Python integer is larger than any finite C double. * Replace with little doubles * that give the same outcome -- w is so large that * its magnitude must exceed the magnitude of any * finite float. */ - if (nbits64 == (uint64_t)-1 && PyErr_Occurred()) { - /* This Python integer is so large that uint64_t isn't - * big enough to hold the # of bits. */ - PyErr_Clear(); - } i = (double)vsign; assert(wsign != 0); j = wsign * 2.0; diff --git a/Objects/longobject.c b/Objects/longobject.c index d34c8b6d71ab3f..9beb5884a6932b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -133,8 +133,16 @@ long_normalize(PyLongObject *v) /* Allocate a new int object with size digits. Return NULL and set exception if we run out of memory. */ -#define MAX_LONG_DIGITS \ +#if SIZEOF_SIZE_T < 8 +# define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, long_value.ob_digit))/sizeof(digit)) +#else +/* Guarantee that the number of bits fits in int64_t. + This is more than an exbibyte, that is more than many of modern + architectures support in principle. + -1 is added to avoid overflow in _PyLong_Frexp(). */ +# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) +#endif PyLongObject * _PyLong_New(Py_ssize_t size) @@ -804,11 +812,11 @@ bit_length_digit(digit x) return _Py_bit_length((unsigned long)x); } -uint64_t +int64_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; - uint64_t result = 0; + int64_t result = 0; Py_ssize_t ndigits; int msd_bits; @@ -818,21 +826,12 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->long_value.ob_digit[ndigits - 1]; - if ((uint64_t)(ndigits - 1) > UINT64_MAX / (uint64_t)PyLong_SHIFT) - goto Overflow; - result = (uint64_t)(ndigits - 1) * (uint64_t)PyLong_SHIFT; + assert(ndigits <= INT64_MAX / PyLong_SHIFT); + result = (int64_t)(ndigits - 1) * PyLong_SHIFT; msd_bits = bit_length_digit(msd); - if (UINT64_MAX - msd_bits < result) - goto Overflow; result += msd_bits; } return result; - - Overflow: - /* Very unlikely. Such integer would require more than 2 exbibytes of RAM. */ - PyErr_SetString(PyExc_OverflowError, "int has too many bits " - "to express in a 64-bit integer"); - return (uint64_t)-1; } PyObject * @@ -1247,15 +1246,12 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) /* Calculates the number of bits required for the *absolute* value * of v. This does not take sign into account, only magnitude. */ - uint64_t nb = _PyLong_NumBits((PyObject *)v); - if (nb == (uint64_t)-1) { - res = -1; - } else { - /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up - * multiples of 8 to the next byte, but we add an implied bit for - * the sign and it cancels out. */ - res = (Py_ssize_t)(nb / 8) + 1; - } + int64_t nb = _PyLong_NumBits((PyObject *)v); + assert(nb >= 0); + /* Normally this would be ((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; /* Two edge cases exist that are best handled after extracting the * bits. These may result in us reporting overflow when the value @@ -3415,7 +3411,8 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) double _PyLong_Frexp(PyLongObject *a, int64_t *e) { - Py_ssize_t a_size, shift_digits, shift_bits, x_size; + Py_ssize_t a_size, shift_digits, x_size; + int shift_bits; int64_t a_bits; /* See below for why x_digits is always large enough. */ digit rem; @@ -3432,14 +3429,7 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) *e = 0; return 0.0; } - int msd_bits = bit_length_digit(a->long_value.ob_digit[a_size-1]); - /* The following is an overflow-free version of the check - "if ((a_size - 1) * PyLong_SHIFT + msd_bits > PY_SSIZE_T_MAX) ..." */ - if (a_size >= (INT64_MAX - 1) / PyLong_SHIFT + 1 && - (a_size > (INT64_MAX - 1) / PyLong_SHIFT + 1 || - msd_bits > (INT64_MAX - 1) % PyLong_SHIFT + 1)) - goto overflow; - a_bits = (int64_t)(a_size - 1) * PyLong_SHIFT + msd_bits; + a_bits = _PyLong_NumBits((PyObject *)a); /* Shift the first DBL_MANT_DIG + 2 bits of a into x_digits[0:x_size] (shifting left if a_bits <= DBL_MANT_DIG + 2). @@ -3468,18 +3458,18 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) */ if (a_bits <= DBL_MANT_DIG + 2) { shift_digits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) / PyLong_SHIFT; - shift_bits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) % PyLong_SHIFT; + shift_bits = (DBL_MANT_DIG + 2 - (int)a_bits) % PyLong_SHIFT; x_size = shift_digits; rem = v_lshift(x_digits + x_size, a->long_value.ob_digit, a_size, - (int)shift_bits); + shift_bits); x_size += a_size; x_digits[x_size++] = rem; } else { shift_digits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT); - shift_bits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); + shift_bits = (int)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); rem = v_rshift(x_digits, a->long_value.ob_digit + shift_digits, - a_size - shift_digits, (int)shift_bits); + a_size - shift_digits, shift_bits); x_size = a_size - shift_digits; /* For correct rounding below, we need the least significant bit of x to be 'sticky' for this shift: if any of the bits @@ -3505,21 +3495,13 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) /* Rescale; make correction if result is 1.0. */ dx /= 4.0 * EXP2_DBL_MANT_DIG; if (dx == 1.0) { - if (a_bits == INT64_MAX) - goto overflow; + assert(a_bits < INT64_MAX); dx = 0.5; a_bits += 1; } *e = a_bits; return _PyLong_IsNegative(a) ? -dx : dx; - - overflow: - /* exponent > PY_SSIZE_T_MAX */ - PyErr_SetString(PyExc_OverflowError, - "huge integer: number of bits overflows a Py_ssize_t"); - *e = 0; - return -1.0; } /* Get a C double from an int object. Rounds to the nearest double, @@ -3547,7 +3529,9 @@ PyLong_AsDouble(PyObject *v) return (double)medium_value((PyLongObject *)v); } x = _PyLong_Frexp((PyLongObject *)v, &exponent); - if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { + assert(exponent >= 0); + assert(!PyErr_Occurred()); + if (exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, "int too large to convert to float"); return -1.0; @@ -5217,39 +5201,6 @@ long_bool(PyLongObject *v) return !_PyLong_IsZero(v); } -/* wordshift, remshift = divmod(shiftby, PyLong_SHIFT) */ -static int -divmod_shift(PyObject *shiftby, Py_ssize_t *wordshift, digit *remshift) -{ - assert(PyLong_Check(shiftby)); - assert(!_PyLong_IsNegative((PyLongObject *)shiftby)); - Py_ssize_t lshiftby = PyLong_AsSsize_t((PyObject *)shiftby); - if (lshiftby >= 0) { - *wordshift = lshiftby / PyLong_SHIFT; - *remshift = lshiftby % PyLong_SHIFT; - return 0; - } - /* PyLong_Check(shiftby) is true and shiftby is not negative, so it must - be that PyLong_AsSsize_t raised an OverflowError. */ - assert(PyErr_ExceptionMatches(PyExc_OverflowError)); - PyErr_Clear(); - PyLongObject *wordshift_obj = divrem1((PyLongObject *)shiftby, PyLong_SHIFT, remshift); - if (wordshift_obj == NULL) { - return -1; - } - *wordshift = PyLong_AsSsize_t((PyObject *)wordshift_obj); - Py_DECREF(wordshift_obj); - if (*wordshift >= 0 && *wordshift < PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(digit)) { - return 0; - } - PyErr_Clear(); - /* Clip the value. With such large wordshift the right shift - returns 0 and the left shift raises an error in _PyLong_New(). */ - *wordshift = PY_SSIZE_T_MAX / sizeof(digit); - *remshift = 0; - return 0; -} - /* Inner function for both long_rshift and _PyLong_Rshift, shifting an integer right by PyLong_SHIFT*wordshift + remshift bits. wordshift should be nonnegative. */ @@ -5343,8 +5294,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_rshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5355,24 +5305,35 @@ long_rshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) - return NULL; - return long_rshift1((PyLongObject *)a, wordshift, remshift); + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + PyErr_Clear(); + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } + return _PyLong_Rshift(a, shiftby); } /* Return a >> shiftby. */ PyObject * -_PyLong_Rshift(PyObject *a, uint64_t shiftby) +_PyLong_Rshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { if (_PyLong_IsNegative((PyLongObject *)a)) { return PyLong_FromLong(-1); } @@ -5430,8 +5391,7 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_lshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5442,24 +5402,30 @@ long_lshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + } return NULL; - return long_lshift1((PyLongObject *)a, wordshift, remshift); + } + return _PyLong_Lshift(a, shiftby); } /* Return a << shiftby. */ PyObject * -_PyLong_Lshift(PyObject *a, uint64_t shiftby) +_PyLong_Lshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); return NULL; @@ -6213,11 +6179,10 @@ static PyObject * int_bit_length_impl(PyObject *self) /*[clinic end generated code: output=fc1977c9353d6a59 input=e4eb7a587e849a32]*/ { - uint64_t nbits = _PyLong_NumBits(self); - if (nbits == (uint64_t)-1) { - return NULL; - } - return PyLong_FromUnsignedLongLong(nbits); + int64_t nbits = _PyLong_NumBits(self); + assert(nbits >= 0); + assert(!PyErr_Occurred()); + return PyLong_FromInt64(nbits); } static int @@ -6251,40 +6216,13 @@ int_bit_count_impl(PyObject *self) PyLongObject *z = (PyLongObject *)self; Py_ssize_t ndigits = _PyLong_DigitCount(z); - Py_ssize_t bit_count = 0; + int64_t bit_count = 0; - /* Each digit has up to PyLong_SHIFT ones, so the accumulated bit count - from the first PY_SSIZE_T_MAX/PyLong_SHIFT digits can't overflow a - Py_ssize_t. */ - Py_ssize_t ndigits_fast = Py_MIN(ndigits, PY_SSIZE_T_MAX/PyLong_SHIFT); - for (Py_ssize_t i = 0; i < ndigits_fast; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { bit_count += popcount_digit(z->long_value.ob_digit[i]); } - PyObject *result = PyLong_FromSsize_t(bit_count); - if (result == NULL) { - return NULL; - } - - /* Use Python integers if bit_count would overflow. */ - for (Py_ssize_t i = ndigits_fast; i < ndigits; i++) { - PyObject *x = PyLong_FromLong(popcount_digit(z->long_value.ob_digit[i])); - if (x == NULL) { - goto error; - } - PyObject *y = long_add((PyLongObject *)result, (PyLongObject *)x); - Py_DECREF(x); - if (y == NULL) { - goto error; - } - Py_SETREF(result, y); - } - - return result; - - error: - Py_DECREF(result); - return NULL; + return PyLong_FromInt64(bit_count); } /*[clinic input] diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1318ce0319d438..2942ab624edf72 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -143,14 +143,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw) static PyObject * -range_vectorcall(PyTypeObject *type, PyObject *const *args, +range_vectorcall(PyObject *rangetype, PyObject *const *args, size_t nargsf, PyObject *kwnames) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (!_PyArg_NoKwnames("range", kwnames)) { return NULL; } - return range_from_array(type, args, nargs); + return range_from_array((PyTypeObject *)rangetype, args, nargs); } PyDoc_STRVAR(range_doc, @@ -803,7 +803,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_init */ 0, /* tp_alloc */ range_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)range_vectorcall + .tp_vectorcall = range_vectorcall }; /*********************** range Iterator **************************/ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f14f10ab9c0a46..4d8cca68df946a 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -999,8 +999,9 @@ tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) } static PyObject * -tupleiter_next(_PyTupleIterObject *it) +tupleiter_next(PyObject *obj) { + _PyTupleIterObject *it = (_PyTupleIterObject *)obj; PyTupleObject *seq; PyObject *item; @@ -1101,7 +1102,7 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tupleiter_next, /* tp_iternext */ + tupleiter_next, /* tp_iternext */ tupleiter_methods, /* tp_methods */ 0, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 68e481f8e5163b..6484e8921f8122 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1435,6 +1435,9 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } @@ -3929,7 +3932,7 @@ type_new_alloc(type_new_ctx *ctx) et->ht_token = NULL; #ifdef Py_GIL_DISABLED - _PyType_AssignId(et); + et->unique_id = _PyObject_AssignUniqueId((PyObject *)et); #endif return type; @@ -5023,7 +5026,7 @@ PyType_FromMetaclass( #ifdef Py_GIL_DISABLED // Assign a type id to enable thread-local refcounting - _PyType_AssignId(res); + res->unique_id = _PyObject_AssignUniqueId((PyObject *)res); #endif /* Ready the type (which includes inheritance). @@ -5207,8 +5210,8 @@ PyType_GetModuleState(PyTypeObject *type) /* Get the module of the first superclass where the module has the * given PyModuleDef. */ -static inline PyObject * -get_module_by_def(PyTypeObject *type, PyModuleDef *def) +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); @@ -5241,7 +5244,7 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); - if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + if (!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped continue; } @@ -5254,37 +5257,14 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) } } END_TYPE_LOCK(); - return res; -} -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *module = get_module_by_def(type, def); - if (module == NULL) { + if (res == NULL) { PyErr_Format( PyExc_TypeError, "PyType_GetModuleByDef: No superclass of '%s' has the given module", type->tp_name); } - return module; -} - -PyObject * -_PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, - PyModuleDef *def) -{ - PyObject *module = get_module_by_def(left, def); - if (module == NULL) { - module = get_module_by_def(right, def); - if (module == NULL) { - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' nor '%s' has " - "the given module", left->tp_name, right->tp_name); - } - } - return module; + return res; } @@ -6100,7 +6080,7 @@ type_dealloc(PyObject *self) Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); #ifdef Py_GIL_DISABLED - _PyType_ReleaseId(et); + _PyObject_ReleaseUniqueId(et->unique_id); #endif et->ht_token = NULL; Py_TYPE(type)->tp_free((PyObject *)type); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 09e9ab39364742..51d93ed8b5ba8c 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 3) { // SOURCE + if (format == 3) { // STRING PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; @@ -1915,7 +1915,16 @@ typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, return NULL; } ta->name = Py_NewRef(name); - ta->type_params = Py_IsNone(type_params) ? NULL : Py_XNewRef(type_params); + if ( + type_params == NULL + || Py_IsNone(type_params) + || (PyTuple_Check(type_params) && PyTuple_GET_SIZE(type_params) == 0) + ) { + ta->type_params = NULL; + } + else { + ta->type_params = Py_NewRef(type_params); + } ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); ta->module = Py_XNewRef(module); diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index 503f3193e2803e..010f5fe5646630 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -169,9 +169,9 @@ WIN32 is still required for the locale module. #endif /* MS_WIN64 */ /* set the version macros for the windows headers */ -/* Python 3.9+ requires Windows 8 or greater */ -#define Py_WINVER 0x0602 /* _WIN32_WINNT_WIN8 */ -#define Py_NTDDI NTDDI_WIN8 +/* Python 3.13+ requires Windows 10 or greater */ +#define Py_WINVER 0x0A00 /* _WIN32_WINNT_WIN10 */ +#define Py_NTDDI NTDDI_WIN10 /* We only set these values when building Python - we don't want to force these values on extensions, as that will affect the prototypes and diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 743e6e2a66a8f1..a3c2d32c454e04 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -268,7 +268,7 @@ - + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 0887a47917a931..91b1d75fb8df5e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -467,7 +467,7 @@ Source Files - + Source Files diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index a1409ecf043d2d..846e027e10c7fa 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,6 +97,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index e27e3171e1e6aa..57be2e2fc5b950 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,6 +12,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 19b982db7f5b87..3b33c6bf6bb91d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -304,13 +304,13 @@ - + @@ -657,7 +657,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 23f2e9c8bc0eb7..ee2930b10439a9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -831,9 +831,6 @@ Include\internal - - Include\internal - Include\internal @@ -846,6 +843,9 @@ Include\internal + + Include\internal + Include\internal\mimalloc @@ -1499,7 +1499,7 @@ Python - + Python diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 10ee6b7be23e21..ab2b2d06cca15d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -810,6 +810,7 @@ static void set_most_env_vars(void) #ifdef Py_STATS putenv("PYTHONSTATS=1"); #endif + putenv("PYTHONPERFSUPPORT=1"); } @@ -1844,6 +1845,10 @@ static int test_initconfig_api(void) goto error; } + if (PyInitConfig_SetInt(config, "perf_profiling", 2) < 0) { + goto error; + } + // Set a UTF-8 string (program_name) if (PyInitConfig_SetStr(config, "program_name", PROGRAM_NAME_UTF8) < 0) { goto error; diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f5b04757e08bf3..01e208b88eca8b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -169,11 +169,10 @@ safe_multiply(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); - uint64_t wbits = _PyLong_NumBits(w); - if (vbits == (uint64_t)-1 || wbits == (uint64_t)-1) { - return NULL; - } + int64_t vbits = _PyLong_NumBits(v); + int64_t wbits = _PyLong_NumBits(w); + assert(vbits >= 0); + assert(wbits >= 0); if (vbits + wbits > MAX_INT_SIZE) { return NULL; } @@ -215,12 +214,13 @@ safe_power(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && _PyLong_IsPositive((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (vbits > MAX_INT_SIZE / wbits) { + if ((uint64_t)vbits > MAX_INT_SIZE / wbits) { return NULL; } } @@ -234,12 +234,13 @@ safe_lshift(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (wbits > MAX_INT_SIZE || vbits > MAX_INT_SIZE - wbits) { + if (wbits > MAX_INT_SIZE || (uint64_t)vbits > MAX_INT_SIZE - wbits) { return NULL; } } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0fd396f1319e78..c712c772201e10 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2254,7 +2254,6 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *old_value; - uint64_t new_version; DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); @@ -2264,9 +2263,8 @@ dummy_func( } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; - new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); ep->me_value = PyStackRef_AsPyObjectSteal(value); - dict->ma_version_tag = new_version; // PEP 509 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -4836,6 +4834,14 @@ dummy_func( assert(((_PyExecutorObject *)executor)->vm_data.valid); } + tier2 op(_MAKE_WARM, (--)) { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + } + tier2 op(_FATAL_ERROR, (--)) { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6f4476d055b5ec..1d9381d09dfb62 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1289,6 +1289,12 @@ _Py_HandlePending(PyThreadState *tstate) _Py_RunGC(tstate); } + if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + _Py_Executors_InvalidateCold(tstate->interp); + tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + } + /* GIL drop request */ if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 8dae747c44a543..9b3373bc1a67a5 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -36,6 +36,33 @@ monitoring_use_tool_id(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(monitoring_clear_tool_id__doc__, +"clear_tool_id($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_CLEAR_TOOL_ID_METHODDEF \ + {"clear_tool_id", (PyCFunction)monitoring_clear_tool_id, METH_O, monitoring_clear_tool_id__doc__}, + +static PyObject * +monitoring_clear_tool_id_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_clear_tool_id(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_clear_tool_id_impl(module, tool_id); + +exit: + return return_value; +} + PyDoc_STRVAR(monitoring_free_tool_id__doc__, "free_tool_id($module, tool_id, /)\n" "--\n" @@ -304,4 +331,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=14ffc0884a6de50a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f81876c6aba9be8 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index 9c0a3fad314cb5..68dc232bb86163 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,6 +16,12 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +static const char *codecs_builtin_error_handlers[] = { + "strict", "ignore", "replace", + "xmlcharrefreplace", "backslashreplace", "namereplace", + "surrogatepass", "surrogateescape", +}; + const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ @@ -618,6 +624,20 @@ int PyCodec_RegisterError(const char *name, PyObject *error) name, error); } +int _PyCodec_UnregisterError(const char *name) +{ + for (size_t i = 0; i < Py_ARRAY_LENGTH(codecs_builtin_error_handlers); ++i) { + if (strcmp(name, codecs_builtin_error_handlers[i]) == 0) { + PyErr_Format(PyExc_ValueError, + "cannot un-register built-in error handler '%s'", name); + return -1; + } + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->codecs.initialized); + return PyDict_PopString(interp->codecs.error_registry, name, NULL); +} + /* Lookup the error handling callback function registered under the name error. As a special case NULL can be passed, in which case the error handling callback for strict encoding will be returned. */ @@ -1470,6 +1490,8 @@ _PyCodec_InitRegistry(PyInterpreterState *interp) } } }; + // ensure that the built-in error handlers' names are kept in sync + assert(Py_ARRAY_LENGTH(methods) == Py_ARRAY_LENGTH(codecs_builtin_error_handlers)); assert(interp->codecs.initialized == 0); interp->codecs.search_path = PyList_New(0); diff --git a/Python/compile.c b/Python/compile.c index 7b3e6f336e44b1..9826d3fbbde976 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -911,7 +911,17 @@ PyObject * _PyCompile_StaticAttributesAsTuple(compiler *c) { assert(c->u->u_static_attributes); - return PySequence_Tuple(c->u->u_static_attributes); + PyObject *static_attributes_unsorted = PySequence_List(c->u->u_static_attributes); + if (static_attributes_unsorted == NULL) { + return NULL; + } + if (PyList_Sort(static_attributes_unsorted) != 0) { + Py_DECREF(static_attributes_unsorted); + return NULL; + } + PyObject *static_attributes = PySequence_Tuple(static_attributes_unsorted); + Py_DECREF(static_attributes_unsorted); + return static_attributes; } int diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7a9c6ab89c38cc..fdfec66b73c730 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2625,7 +2625,6 @@ JUMP_TO_JUMP_TARGET(); } PyObject *old_value; - uint64_t new_version; if (!DK_IS_UNICODE(dict->ma_keys)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2641,9 +2640,8 @@ } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; - new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); ep->me_value = PyStackRef_AsPyObjectSteal(value); - dict->ma_version_tag = new_version; // PEP 509 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -5435,6 +5433,15 @@ break; } + case _MAKE_WARM: { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + break; + } + case _FATAL_ERROR: { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/gc.c b/Python/gc.c index 024d041437be4a..028657eb8999c1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1944,6 +1944,13 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } } +static void +finalize_unlink_gc_head(PyGC_Head *gc) { + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); +} void _PyGC_Fini(PyInterpreterState *interp) @@ -1952,9 +1959,25 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ + /* Prevent a subtle bug that affects sub-interpreters that use basic + * single-phase init extensions (m_size == -1). Those extensions cause objects + * to be shared between interpreters, via the PyDict_Update(mdict, m_copy) call + * in import_find_extension(). + * + * If they are GC objects, their GC head next or prev links could refer to + * the interpreter _gc_runtime_state PyGC_Head nodes. Those nodes go away + * when the interpreter structure is freed and so pointers to them become + * invalid. If those objects are still used by another interpreter and + * UNTRACK is called on them, a crash will happen. We untrack the nodes + * here to avoid that. + * + * This bug was originally fixed when reported as gh-90228. The bug was + * re-introduced in gh-94673. + */ + finalize_unlink_gc_head(&gcstate->young.head); + finalize_unlink_gc_head(&gcstate->old[0].head); + finalize_unlink_gc_head(&gcstate->old[1].head); + finalize_unlink_gc_head(&gcstate->permanent_generation.head); } /* for debugging */ diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index a5bc9b9b5782b2..38564d9d9b0058 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -15,7 +15,7 @@ #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" -#include "pycore_typeid.h" // _PyType_MergeThreadLocalRefcounts +#include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts #ifdef Py_GIL_DISABLED @@ -217,12 +217,12 @@ disable_deferred_refcounting(PyObject *op) merge_refcount(op, 0); } - // Heap types also use thread-local refcounting -- disable it here. + // Heap types also use per-thread refcounting -- disable it here. if (PyType_Check(op)) { - // Disable thread-local refcounting for heap types - PyTypeObject *type = (PyTypeObject *)op; - if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - _PyType_ReleaseId((PyHeapTypeObject *)op); + if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) { + PyHeapTypeObject *ht = (PyHeapTypeObject *)op; + _PyObject_ReleaseUniqueId(ht->unique_id); + ht->unique_id = -1; } } @@ -1221,7 +1221,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; // merge per-thread refcount for types into the type's actual refcount - _PyType_MergeThreadLocalRefcounts(tstate); + _PyObject_MergePerThreadRefcounts(tstate); // merge refcounts for all queued objects merge_queued_objects(tstate, state); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1201fe82efb919..9de7554d4dfd55 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6940,7 +6940,6 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; - uint64_t new_version; DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); @@ -6950,9 +6949,8 @@ } old_value = ep->me_value; PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED; - new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value)); ep->me_value = PyStackRef_AsPyObjectSteal(value); - dict->ma_version_tag = new_version; // PEP 509 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); diff --git a/Python/initconfig.c b/Python/initconfig.c index d93244f7f41084..58ac5e7d7eaeff 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -150,7 +150,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), - SPEC(perf_profiling, BOOL, READ_ONLY, NO_SYS), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), SPEC(program_name, WSTR, READ_ONLY, NO_SYS), SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5e51a9c992f6c2..8fd7c08beac92a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1660,6 +1660,16 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) if (allocate_instrumentation_data(code)) { return -1; } + // If the local monitors are out of date, clear them up + _Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors; + for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { + if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) { + for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) { + local_monitors->tools[j] &= ~(1 << i); + } + } + } + _Py_LocalMonitors all_events = local_union( interp->monitors, code->_co_monitoring->local_monitors); @@ -2004,6 +2014,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent goto done; } + code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id]; + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { @@ -2036,6 +2048,43 @@ _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent return 0; } +int _PyMonitoring_ClearToolId(int tool_id) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_GET(); + + for (int i = 0; i < _PY_MONITORING_EVENTS; i++) { + PyObject *func = _PyMonitoring_RegisterCallback(tool_id, i, NULL); + if (func != NULL) { + Py_DECREF(func); + } + } + + if (_PyMonitoring_SetEvents(tool_id, 0) < 0) { + return -1; + } + + _PyEval_StopTheWorld(interp); + uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT; + if (version == 0) { + PyErr_Format(PyExc_OverflowError, "events set too many times"); + _PyEval_StartTheWorld(interp); + return -1; + } + + // monitoring_tool_versions[tool_id] is set to latest global version here to + // 1. invalidate local events on all existing code objects + // 2. be ready for the next call to set local events + interp->monitoring_tool_versions[tool_id] = version; + + // Set the new global version so all the code objects can refresh the + // instrumentation. + set_global_version(_PyThreadState_GET(), version); + int res = instrument_all_executing_code_objects(interp); + _PyEval_StartTheWorld(interp); + return res; +} + /*[clinic input] module monitoring [clinic start generated code]*/ @@ -2083,6 +2132,33 @@ monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) Py_RETURN_NONE; } +/*[clinic input] +monitoring.clear_tool_id + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_clear_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=04defc23470b1be7 input=af643d6648a66163]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + + PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (interp->monitoring_tool_names[tool_id] != NULL) { + if (_PyMonitoring_ClearToolId(tool_id) < 0) { + return NULL; + } + } + + Py_RETURN_NONE; +} + /*[clinic input] monitoring.free_tool_id @@ -2099,6 +2175,13 @@ monitoring_free_tool_id_impl(PyObject *module, int tool_id) return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (interp->monitoring_tool_names[tool_id] != NULL) { + if (_PyMonitoring_ClearToolId(tool_id) < 0) { + return NULL; + } + } + Py_CLEAR(interp->monitoring_tool_names[tool_id]); Py_RETURN_NONE; } @@ -2376,6 +2459,7 @@ monitoring__all_events_impl(PyObject *module) static PyMethodDef methods[] = { MONITORING_USE_TOOL_ID_METHODDEF + MONITORING_CLEAR_TOOL_ID_METHODDEF MONITORING_FREE_TOOL_ID_METHODDEF MONITORING_GET_TOOL_METHODDEF MONITORING_REGISTER_CALLBACK_METHODDEF diff --git a/Python/optimizer.c b/Python/optimizer.c index bb7a90b3204f40..978649faa04d45 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -565,6 +565,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; for (;;) { @@ -1194,6 +1195,9 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil executor->jit_code = NULL; executor->jit_side_entry = NULL; executor->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + executor->vm_data.warm = true; if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); return NULL; @@ -1659,4 +1663,42 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) } } +void +_Py_Executors_InvalidateCold(PyInterpreterState *interp) +{ + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + goto error; + } + + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + + if (!exec->vm_data.warm && PyList_Append(invalidate, (PyObject *)exec) < 0) { + goto error; + } + else { + exec->vm_data.warm = false; + } + + exec = next; + } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback + _Py_Executors_InvalidateAll(interp, 0); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index b202b58a8b7214..f30e873605d858 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -56,14 +56,14 @@ static int get_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); + return (d->_ma_watcher_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); } static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - d->ma_version_tag += (1 << DICT_MAX_WATCHERS); + d->_ma_watcher_tag += (1 << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a6cfa271ae6758..4d172e3c762704 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2381,6 +2381,10 @@ break; } + case _MAKE_WARM: { + break; + } + case _FATAL_ERROR: { break; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 27faf723745c21..d9e89edf5ddc9e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -28,7 +28,7 @@ #include "pycore_sliceobject.h" // _PySlice_Fini() #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() -#include "pycore_typeid.h" // _PyType_FinalizeIdPool() +#include "pycore_uniqueid.h" // _PyType_FinalizeIdPool() #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() @@ -1834,7 +1834,7 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_Fini(interp); #ifdef Py_GIL_DISABLED - _PyType_FinalizeIdPool(interp); + _PyObject_FinalizeUniqueIdPool(interp); #endif _PyCode_Fini(interp); @@ -2503,18 +2503,12 @@ finalize_subinterpreters(void) static PyStatus add_main_module(PyInterpreterState *interp) { - PyObject *m, *d, *ann_dict; + PyObject *m, *d; m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); d = PyModule_GetDict(m); - ann_dict = PyDict_New(); - if ((ann_dict == NULL) || - (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) { - return _PyStatus_ERR("Failed to initialize __main__.__annotations__"); - } - Py_DECREF(ann_dict); int has_builtins = PyDict_ContainsString(d, "__builtins__"); if (has_builtins < 0) { diff --git a/Python/pystate.c b/Python/pystate.c index 6bf7ebeb75ff73..45e79ade7b6035 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -20,7 +20,7 @@ #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() -#include "pycore_typeid.h" // _PyType_FinalizeThreadLocalRefcounts() +#include "pycore_uniqueid.h" // _PyType_FinalizeThreadLocalRefcounts() /* -------------------------------------------------------------------------- CAUTION @@ -654,12 +654,14 @@ init_interpreter(PyInterpreterState *interp, interp->monitoring_callables[t][e] = NULL; } + interp->monitoring_tool_versions[t] = 0; } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; #ifdef _Py_TIER2 (void)_Py_SetOptimizer(interp, NULL); interp->executor_list_head = NULL; + interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; #endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ @@ -1744,7 +1746,7 @@ PyThreadState_Clear(PyThreadState *tstate) // Merge our thread-local refcounts into the type's own refcount and // free our local refcount array. - _PyType_FinalizeThreadLocalRefcounts((_PyThreadStateImpl *)tstate); + _PyObject_FinalizePerThreadRefcounts((_PyThreadStateImpl *)tstate); // Remove ourself from the biased reference counting table of threads. _Py_brc_remove_thread(tstate); @@ -1804,7 +1806,7 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; tstate->interp->object_state.reftotal += tstate_impl->reftotal; tstate_impl->reftotal = 0; - assert(tstate_impl->types.refcounts == NULL); + assert(tstate_impl->refcounts.values == NULL); #endif HEAD_UNLOCK(runtime); diff --git a/Python/typeid.c b/Python/uniqueid.c similarity index 51% rename from Python/typeid.c rename to Python/uniqueid.c index 83a68723ded61b..9a9ee2f39467b0 100644 --- a/Python/typeid.c +++ b/Python/uniqueid.c @@ -3,12 +3,14 @@ #include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_object.h" // _Py_IncRefTotal -#include "pycore_typeid.h" +#include "pycore_uniqueid.h" -// This contains code for allocating unique ids to heap type objects -// and re-using those ids when the type is deallocated. +// This contains code for allocating unique ids for per-thread reference +// counting and re-using those ids when an object is deallocated. // -// See Include/internal/pycore_typeid.h for more details. +// Currently, per-thread reference counting is only used for heap types. +// +// See Include/internal/pycore_uniqueid.h for more details. #ifdef Py_GIL_DISABLED @@ -18,7 +20,7 @@ #define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) static int -resize_interp_type_id_pool(struct _Py_type_id_pool *pool) +resize_interp_type_id_pool(struct _Py_unique_id_pool *pool) { if ((size_t)pool->size > PY_SSIZE_T_MAX / (2 * sizeof(*pool->table))) { return -1; @@ -29,8 +31,8 @@ resize_interp_type_id_pool(struct _Py_type_id_pool *pool) new_size = POOL_MIN_SIZE; } - _Py_type_id_entry *table = PyMem_Realloc(pool->table, - new_size * sizeof(*pool->table)); + _Py_unique_id_entry *table = PyMem_Realloc(pool->table, + new_size * sizeof(*pool->table)); if (table == NULL) { return -1; } @@ -50,70 +52,67 @@ resize_interp_type_id_pool(struct _Py_type_id_pool *pool) static int resize_local_refcounts(_PyThreadStateImpl *tstate) { - if (tstate->types.is_finalized) { + if (tstate->refcounts.is_finalized) { return -1; } - struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; + struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids; Py_ssize_t size = _Py_atomic_load_ssize(&pool->size); - Py_ssize_t *refcnts = PyMem_Realloc(tstate->types.refcounts, + Py_ssize_t *refcnts = PyMem_Realloc(tstate->refcounts.values, size * sizeof(Py_ssize_t)); if (refcnts == NULL) { return -1; } - Py_ssize_t old_size = tstate->types.size; + Py_ssize_t old_size = tstate->refcounts.size; if (old_size < size) { memset(refcnts + old_size, 0, (size - old_size) * sizeof(Py_ssize_t)); } - tstate->types.refcounts = refcnts; - tstate->types.size = size; + tstate->refcounts.values = refcnts; + tstate->refcounts.size = size; return 0; } -void -_PyType_AssignId(PyHeapTypeObject *type) +Py_ssize_t +_PyObject_AssignUniqueId(PyObject *obj) { PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_type_id_pool *pool = &interp->type_ids; + struct _Py_unique_id_pool *pool = &interp->unique_ids; LOCK_POOL(pool); if (pool->freelist == NULL) { if (resize_interp_type_id_pool(pool) < 0) { - type->unique_id = -1; UNLOCK_POOL(pool); - return; + return -1; } } - _Py_type_id_entry *entry = pool->freelist; + _Py_unique_id_entry *entry = pool->freelist; pool->freelist = entry->next; - entry->type = type; - _PyObject_SetDeferredRefcount((PyObject *)type); - type->unique_id = (entry - pool->table); + entry->obj = obj; + _PyObject_SetDeferredRefcount(obj); + Py_ssize_t unique_id = (entry - pool->table); UNLOCK_POOL(pool); + return unique_id; } void -_PyType_ReleaseId(PyHeapTypeObject *type) +_PyObject_ReleaseUniqueId(Py_ssize_t unique_id) { PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _Py_type_id_pool *pool = &interp->type_ids; + struct _Py_unique_id_pool *pool = &interp->unique_ids; - if (type->unique_id < 0) { - // The type doesn't have an id assigned. + if (unique_id < 0) { + // The id is not assigned return; } LOCK_POOL(pool); - _Py_type_id_entry *entry = &pool->table[type->unique_id]; - assert(entry->type == type); + _Py_unique_id_entry *entry = &pool->table[unique_id]; entry->next = pool->freelist; pool->freelist = entry; - - type->unique_id = -1; UNLOCK_POOL(pool); } @@ -127,8 +126,8 @@ _PyType_IncrefSlow(PyHeapTypeObject *type) return; } - assert(type->unique_id < tstate->types.size); - tstate->types.refcounts[type->unique_id]++; + assert(type->unique_id < tstate->refcounts.size); + tstate->refcounts.values[type->unique_id]++; #ifdef Py_REF_DEBUG _Py_IncRefTotal((PyThreadState *)tstate); #endif @@ -136,59 +135,64 @@ _PyType_IncrefSlow(PyHeapTypeObject *type) } void -_PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate) +_PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate) { - if (tstate->types.refcounts == NULL) { + if (tstate->refcounts.values == NULL) { return; } - struct _Py_type_id_pool *pool = &tstate->base.interp->type_ids; + struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids; LOCK_POOL(pool); - for (Py_ssize_t i = 0, n = tstate->types.size; i < n; i++) { - Py_ssize_t refcnt = tstate->types.refcounts[i]; + for (Py_ssize_t i = 0, n = tstate->refcounts.size; i < n; i++) { + Py_ssize_t refcnt = tstate->refcounts.values[i]; if (refcnt != 0) { - PyObject *type = (PyObject *)pool->table[i].type; - assert(PyType_Check(type)); - - _Py_atomic_add_ssize(&type->ob_ref_shared, + PyObject *obj = pool->table[i].obj; + _Py_atomic_add_ssize(&obj->ob_ref_shared, refcnt << _Py_REF_SHARED_SHIFT); - tstate->types.refcounts[i] = 0; + tstate->refcounts.values[i] = 0; } } UNLOCK_POOL(pool); } void -_PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate) +_PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate) { - _PyType_MergeThreadLocalRefcounts(tstate); + _PyObject_MergePerThreadRefcounts(tstate); - PyMem_Free(tstate->types.refcounts); - tstate->types.refcounts = NULL; - tstate->types.size = 0; - tstate->types.is_finalized = 1; + PyMem_Free(tstate->refcounts.values); + tstate->refcounts.values = NULL; + tstate->refcounts.size = 0; + tstate->refcounts.is_finalized = 1; } void -_PyType_FinalizeIdPool(PyInterpreterState *interp) +_PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp) { - struct _Py_type_id_pool *pool = &interp->type_ids; + struct _Py_unique_id_pool *pool = &interp->unique_ids; // First, set the free-list to NULL values while (pool->freelist) { - _Py_type_id_entry *next = pool->freelist->next; - pool->freelist->type = NULL; + _Py_unique_id_entry *next = pool->freelist->next; + pool->freelist->obj = NULL; pool->freelist = next; } // Now everything non-NULL is a type. Set the type's id to -1 in case it // outlives the interpreter. for (Py_ssize_t i = 0; i < pool->size; i++) { - PyHeapTypeObject *ht = pool->table[i].type; - if (ht) { - ht->unique_id = -1; - pool->table[i].type = NULL; + PyObject *obj = pool->table[i].obj; + pool->table[i].obj = NULL; + if (obj == NULL) { + continue; + } + if (PyType_Check(obj)) { + assert(PyType_HasFeature((PyTypeObject *)obj, Py_TPFLAGS_HEAPTYPE)); + ((PyHeapTypeObject *)obj)->unique_id = -1; + } + else { + Py_UNREACHABLE(); } } PyMem_Free(pool->table); diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 882918fafb1edd..b5b6de0e7dc2dc 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -433,7 +433,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': # Give a nice message for common mistakes. # To cover tricky cases (like "\n") we also generate C asserts. raise ValueError( - 'do not use &_PyID or &_Py_STR for one-character latin-1 ' + 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name @@ -442,7 +442,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': overlap = identifiers & set(strings.keys()) if overlap: raise ValueError( - 'do not use both _PyID and _Py_DECLARE_STR for the same string: ' + 'do not use both _Py_ID and _Py_DECLARE_STR for the same string: ' + repr(overlap)) return identifiers, strings diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e1c07f88b963bc..a0be2a0a203f8c 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -423,6 +423,7 @@ Modules/readline.c - libedit_history_start - Modules/_ctypes/cfield.c - formattable - Modules/_ctypes/malloc_closure.c - free_list - +Modules/_cursesmodule.c - curses_global_state - Modules/_curses_panel.c - lop - Modules/_ssl/debughelpers.c _PySSL_keylog_callback lock - Modules/_tkinter.c - quitMainLoop - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index f4dc807198a8ef..e6c599a2ac4a46 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -345,6 +345,7 @@ Python/ast_opt.c fold_unaryop ops - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - +Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LOCATION - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index aabe205125856c..a4ce207703edcd 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -540,6 +540,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyList_FromStackRefSteal", "_PyTuple_FromArraySteal", "_PyTuple_FromStackRefSteal", + "_Py_set_eval_breaker_bit" ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index e37ee943999785..6c7b48f1f37865 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -139,6 +139,9 @@ async def _compile( "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", + # On aarch64 Linux, intrinsics were being emitted and this flag + # was required to disable them. + "-mno-outline-atomics", "-std=c11", *self.args, ] diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 0014204e89d1bb..49f681d3e11d2e 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -123,7 +123,7 @@ Feel free to post at <a href="https://discuss.python.org/c/users/7">discus You must restart your computer to complete the rollback of the software. &Restart Unable to install [WixBundleName] due to an existing install. Use Programs and Features to modify, repair or remove [WixBundleName]. - At least Windows 8.1 or Windows Server 2012 are required to install [WixBundleName] + At least Windows 10 or Windows Server 2016 are required to install [WixBundleName] Visit <a href="https://www.python.org/downloads/">python.org</a> to download an earlier version of Python. Disable path length limit diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index 094ddba4f1ad8f..1e0df5084ff079 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -3086,11 +3086,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; if (IsWindowsServer()) { - if (IsWindowsVersionOrGreater(6, 2, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2012 or later"); + if (IsWindowsVersionOrGreater(10, 0, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2016 or later"); return; + } else if (IsWindowsVersionOrGreater(6, 2, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2012"); } else if (IsWindowsVersionOrGreater(6, 1, 1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected Windows Server 2008 R2"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 1, 0)) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 0, 0)) { @@ -3098,14 +3100,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2003 or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2012 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2016 or later is required to continue installation"); } else { if (IsWindows10OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 10 or later"); return; } else if (IsWindows8Point1OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 8.1"); - return; + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8.1"); } else if (IsWindows8OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8"); } else if (IsWindows7OrGreater()) { @@ -3115,7 +3116,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows XP or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 8.1 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 10 or later is required to continue installation"); } LocGetString(_wixLoc, L"#(loc.FailureOldOS)", &pLocString); diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index e25219a6caf9d4..4a13d368d7c744 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -51,14 +51,6 @@ "test_pdb.msi", "tools.msi", "ucrt.msi", - "Windows6.0-KB2999226-x64.msu", - "Windows6.0-KB2999226-x86.msu", - "Windows6.1-KB2999226-x64.msu", - "Windows6.1-KB2999226-x86.msu", - "Windows8.1-KB2999226-x64.msu", - "Windows8.1-KB2999226-x86.msu", - "Windows8-RT-KB2999226-x64.msu", - "Windows8-RT-KB2999226-x86.msu", ] PATHS = [ "python-{}.exe".format(m.group(0)), diff --git a/Tools/wasm/Setup.local.example b/Tools/wasm/Setup.local.example deleted file mode 100644 index 7b2fb13f6ceef2..00000000000000 --- a/Tools/wasm/Setup.local.example +++ /dev/null @@ -1,13 +0,0 @@ -# Module/Setup.local with reduced stdlib -*disabled* -_asyncio -_bz2 -_decimal -_pickle -pyexpat _elementtree -_sha3 _blake2 -_zoneinfo -xxsubtype - -# cjk codecs -#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw diff --git a/Tools/wasm/build_wasi.sh b/Tools/wasm/build_wasi.sh deleted file mode 100755 index 436306222ce1d0..00000000000000 --- a/Tools/wasm/build_wasi.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/bash - -set -e -x - -# Quick check to avoid running configure just to fail in the end. -if [ -f Programs/python.o ]; then - echo "Can't do an out-of-tree build w/ an in-place build pre-existing (i.e., found Programs/python.o)" >&2 - exit 1 -fi - -if [ ! -f Modules/Setup.local ]; then - touch Modules/Setup.local -fi - -# TODO: check if `make` and `pkgconfig` are installed -# TODO: detect if wasmtime is installed - -# Create the "build" Python. -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -s -j 4 all -export PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` -popd - -# Create the "host"/WASI Python. -export CONFIG_SITE="$(pwd)/Tools/wasm/config.site-wasm32-wasi" -export HOSTRUNNER="wasmtime run --mapdir /::$(pwd) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/builddir/wasi/python.wasm --" - -mkdir -p builddir/wasi -pushd builddir/wasi -../../Tools/wasm/wasi-env \ - ../../configure \ - -C \ - --host=wasm32-unknown-wasi \ - --build=$(../../config.guess) \ - --with-build-python=../build/python -make -s -j 4 all -# Create a helper script for executing the host/WASI Python. -printf "#!/bin/sh\nexec $HOSTRUNNER \"\$@\"\n" > run_wasi.sh -chmod 755 run_wasi.sh -./run_wasi.sh --version -popd - diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 95eda863cb62c6..4c5078a1f675e2 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,6 +1,8 @@ #!/bin/sh set -e +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . + # function usage() { echo "wasi-env - Run command with WASI-SDK"