From 3d8b22454728b6d54cc476cfb59e47ab40f3f527 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 8 Oct 2022 01:24:01 +0100 Subject: [PATCH 01/10] GH-94182: Run the PidfdChildWatcher on the running loop (#94184) There is no reason for this watcher to be attached to any particular loop. This should make it safe to use regardless of the lifetime of the event loop running in the main thread (relative to other loops). Co-authored-by: Yury Selivanov Co-authored-by: Jelle Zijlstra --- Lib/asyncio/unix_events.py | 44 +++++---------- Lib/test/test_asyncio/test_subprocess.py | 54 +++++++++++++++---- ...2-06-24-08-49-47.gh-issue-94182.Wknau0.rst | 1 + 3 files changed, 57 insertions(+), 42 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-06-24-08-49-47.gh-issue-94182.Wknau0.rst diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 96e6d73a759794..0f67b4d469f28c 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -912,10 +912,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __init__(self): - self._loop = None - self._callbacks = {} - def __enter__(self): return self @@ -923,35 +919,22 @@ def __exit__(self, exc_type, exc_value, exc_traceback): pass def is_active(self): - return self._loop is not None and self._loop.is_running() + return True def close(self): - self.attach_loop(None) + pass def attach_loop(self, loop): - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - for pidfd, _, _ in self._callbacks.values(): - self._loop._remove_reader(pidfd) - os.close(pidfd) - self._callbacks.clear() - self._loop = loop + pass def add_child_handler(self, pid, callback, *args): - existing = self._callbacks.get(pid) - if existing is not None: - self._callbacks[pid] = existing[0], callback, args - else: - pidfd = os.pidfd_open(pid) - self._loop._add_reader(pidfd, self._do_wait, pid) - self._callbacks[pid] = pidfd, callback, args + loop = events.get_running_loop() + pidfd = os.pidfd_open(pid) + loop._add_reader(pidfd, self._do_wait, pid, pidfd, callback, args) - def _do_wait(self, pid): - pidfd, callback, args = self._callbacks.pop(pid) - self._loop._remove_reader(pidfd) + def _do_wait(self, pid, pidfd, callback, args): + loop = events.get_running_loop() + loop._remove_reader(pidfd) try: _, status = os.waitpid(pid, 0) except ChildProcessError: @@ -969,12 +952,9 @@ def _do_wait(self, pid): callback(pid, returncode, *args) def remove_child_handler(self, pid): - try: - pidfd, _, _ = self._callbacks.pop(pid) - except KeyError: - return False - self._loop._remove_reader(pidfd) - os.close(pidfd) + # asyncio never calls remove_child_handler() !!! + # The method is no-op but is implemented because + # abstract base classes require it. return True diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 9bc60b9dc2ae2e..6ba889407b802e 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -4,6 +4,7 @@ import sys import unittest import warnings +import functools from unittest import mock import asyncio @@ -30,6 +31,19 @@ 'sys.stdout.buffer.write(data)'))] +@functools.cache +def _has_pidfd_support(): + if not hasattr(os, 'pidfd_open'): + return False + + try: + os.close(os.pidfd_open(os.getpid())) + except OSError: + return False + + return True + + def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -708,17 +722,8 @@ class SubprocessFastWatcherTests(SubprocessWatcherMixin, Watcher = unix_events.FastChildWatcher - def has_pidfd_support(): - if not hasattr(os, 'pidfd_open'): - return False - try: - os.close(os.pidfd_open(os.getpid())) - except OSError: - return False - return True - @unittest.skipUnless( - has_pidfd_support(), + _has_pidfd_support(), "operating system does not support pidfds", ) class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, @@ -751,6 +756,35 @@ async def execute(): mock.call.__exit__(RuntimeError, mock.ANY, mock.ANY), ]) + + @unittest.skipUnless( + _has_pidfd_support(), + "operating system does not support pidfds", + ) + def test_create_subprocess_with_pidfd(self): + async def in_thread(): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate(b"some data") + return proc.returncode, stdout + + async def main(): + # asyncio.Runner did not call asyncio.set_event_loop() + with self.assertRaises(RuntimeError): + asyncio.get_event_loop_policy().get_event_loop() + return await asyncio.to_thread(asyncio.run, in_thread()) + + asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) + try: + with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: + returncode, stdout = runner.run(main()) + self.assertEqual(returncode, 0) + self.assertEqual(stdout, b'some data') + finally: + asyncio.set_child_watcher(None) else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2022-06-24-08-49-47.gh-issue-94182.Wknau0.rst b/Misc/NEWS.d/next/Library/2022-06-24-08-49-47.gh-issue-94182.Wknau0.rst new file mode 100644 index 00000000000000..c7be8640ef1f7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-24-08-49-47.gh-issue-94182.Wknau0.rst @@ -0,0 +1 @@ +run the :class:`asyncio.PidfdChildWatcher` on the running loop, this allows event loops to run subprocesses when there is no default event loop running on the main thread From 8ba9378b168ad330c158a001afca03d6c028d39b Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 8 Oct 2022 05:59:09 +0530 Subject: [PATCH 02/10] GH-98023: Change default child watcher to PidfdChildWatcher on supported systems (#98024) --- Lib/asyncio/unix_events.py | 16 +++++++++++++++- Lib/test/test_asyncio/test_unix_events.py | 14 +++++++++++++- ...2022-10-07-09-52-37.gh-issue-98023.aliEcl.rst | 1 + 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-07-09-52-37.gh-issue-98023.aliEcl.rst diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 0f67b4d469f28c..7fc75cd17ef741 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1403,6 +1403,17 @@ def _do_waitpid(self, loop, expected_pid, callback, args): self._threads.pop(expected_pid) +def can_use_pidfd(): + if not hasattr(os, 'pidfd_open'): + return False + try: + pid = os.getpid() + os.close(os.pidfd_open(pid, 0)) + except OSError: + # blocked by security policy like SECCOMP + return False + return True + class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): """UNIX event loop policy with a watcher for child processes.""" @@ -1415,7 +1426,10 @@ def __init__(self): def _init_watcher(self): with events._lock: if self._watcher is None: # pragma: no branch - self._watcher = ThreadedChildWatcher() + if can_use_pidfd(): + self._watcher = PidfdChildWatcher() + else: + self._watcher = ThreadedChildWatcher() if threading.current_thread() is threading.main_thread(): self._watcher.attach_loop(self._local._loop) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 5bad21ecbae4af..03fb5e649d8e97 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1702,7 +1702,8 @@ def create_policy(self): def test_get_default_child_watcher(self): policy = self.create_policy() self.assertIsNone(policy._watcher) - + unix_events.can_use_pidfd = mock.Mock() + unix_events.can_use_pidfd.return_value = False watcher = policy.get_child_watcher() self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) @@ -1710,6 +1711,17 @@ def test_get_default_child_watcher(self): self.assertIs(watcher, policy.get_child_watcher()) + policy = self.create_policy() + self.assertIsNone(policy._watcher) + unix_events.can_use_pidfd = mock.Mock() + unix_events.can_use_pidfd.return_value = True + watcher = policy.get_child_watcher() + self.assertIsInstance(watcher, asyncio.PidfdChildWatcher) + + self.assertIs(policy._watcher, watcher) + + self.assertIs(watcher, policy.get_child_watcher()) + def test_get_child_watcher_after_set(self): policy = self.create_policy() watcher = asyncio.FastChildWatcher() diff --git a/Misc/NEWS.d/next/Library/2022-10-07-09-52-37.gh-issue-98023.aliEcl.rst b/Misc/NEWS.d/next/Library/2022-10-07-09-52-37.gh-issue-98023.aliEcl.rst new file mode 100644 index 00000000000000..1bfd68d4ac7ca0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-07-09-52-37.gh-issue-98023.aliEcl.rst @@ -0,0 +1 @@ +Change default child watcher to :class:`~asyncio.PidfdChildWatcher` on Linux systems which supports it. Patch by Kumar Aditya. From e82d977eb0b53b0d69509b7080107108e5cfc6f9 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 7 Oct 2022 17:37:46 -0700 Subject: [PATCH 03/10] gh-91052: Add PyDict_Unwatch for unwatching a dictionary (#98055) --- Doc/c-api/dict.rst | 21 ++++++++++++- Doc/whatsnew/3.12.rst | 5 +++ Include/cpython/dictobject.h | 1 + Lib/test/test_capi.py | 60 ++++++++++++++++++++++++++---------- Modules/_testcapimodule.c | 15 +++++++++ Objects/dictobject.c | 41 +++++++++++++++++------- 6 files changed, 115 insertions(+), 28 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 7bebea0c97de5a..e5f28b59a701e0 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -246,17 +246,32 @@ Dictionary Objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + .. versionadded:: 3.12 + .. c:function:: int PyDict_ClearWatcher(int watcher_id) Clear watcher identified by *watcher_id* previously returned from :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. if the given *watcher_id* was never registered.) + .. versionadded:: 3.12 + .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) Mark dictionary *dict* as watched. The callback granted *watcher_id* by :c:func:`PyDict_AddWatcher` will be called when *dict* is modified or - deallocated. + deallocated. Return ``0`` on success or ``-1`` on error. + + .. versionadded:: 3.12 + +.. c:function:: int PyDict_Unwatch(int watcher_id, PyObject *dict) + + Mark dictionary *dict* as no longer watched. The callback granted + *watcher_id* by :c:func:`PyDict_AddWatcher` will no longer be called when + *dict* is modified or deallocated. The dict must previously have been + watched by this watcher. Return ``0`` on success or ``-1`` on error. + + .. versionadded:: 3.12 .. c:type:: PyDict_WatchEvent @@ -264,6 +279,8 @@ Dictionary Objects ``PyDict_EVENT_MODIFIED``, ``PyDict_EVENT_DELETED``, ``PyDict_EVENT_CLONED``, ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCATED``. + .. versionadded:: 3.12 + .. c:type:: int (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) Type of a dict watcher callback function. @@ -289,3 +306,5 @@ Dictionary Objects If the callback returns with an exception set, it must return ``-1``; this exception will be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + + .. versionadded:: 3.12 diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 405de11e716b44..f873974b3e78fe 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -546,6 +546,11 @@ New Features which sets the vectorcall field of a given :c:type:`PyFunctionObject`. (Contributed by Andrew Frost in :gh:`92257`.) +* The C API now permits registering callbacks via :c:func:`PyDict_AddWatcher`, + :c:func:`PyDict_AddWatch` and related APIs to be called whenever a dictionary + is modified. This is intended for use by optimizing interpreters, JIT + compilers, or debuggers. + Porting to Python 3.12 ---------------------- diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index f8a74a597b0ea2..2dff59ef0b8a6b 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -106,3 +106,4 @@ PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id); // Mark given dictionary as "watched" (callback will be called if it is modified) PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); +PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict); diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 19367dfcc1ccbb..17425050ce00c0 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -20,6 +20,7 @@ import weakref from test import support from test.support import MISSING_C_DOCSTRINGS +from test.support import catch_unraisable_exception from test.support import import_helper from test.support import threading_helper from test.support import warnings_helper @@ -1421,6 +1422,9 @@ def assert_events(self, expected): def watch(self, wid, d): _testcapi.watch_dict(wid, d) + def unwatch(self, wid, d): + _testcapi.unwatch_dict(wid, d) + def test_set_new_item(self): d = {} with self.watcher() as wid: @@ -1477,27 +1481,24 @@ def test_dealloc(self): del d self.assert_events(["dealloc"]) + def test_unwatch(self): + d = {} + with self.watcher() as wid: + self.watch(wid, d) + d["foo"] = "bar" + self.unwatch(wid, d) + d["hmm"] = "baz" + self.assert_events(["new:foo:bar"]) + def test_error(self): d = {} - unraisables = [] - def unraisable_hook(unraisable): - unraisables.append(unraisable) with self.watcher(kind=self.ERROR) as wid: self.watch(wid, d) - orig_unraisable_hook = sys.unraisablehook - sys.unraisablehook = unraisable_hook - try: + with catch_unraisable_exception() as cm: d["foo"] = "bar" - finally: - sys.unraisablehook = orig_unraisable_hook + self.assertIs(cm.unraisable.object, d) + self.assertEqual(str(cm.unraisable.exc_value), "boom!") self.assert_events([]) - self.assertEqual(len(unraisables), 1) - unraisable = unraisables[0] - self.assertIs(unraisable.object, d) - self.assertEqual(str(unraisable.exc_value), "boom!") - # avoid leaking reference cycles - del unraisable - del unraisables def test_two_watchers(self): d1 = {} @@ -1522,11 +1523,38 @@ def test_watch_out_of_range_watcher_id(self): with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID 8"): self.watch(8, d) # DICT_MAX_WATCHERS = 8 - def test_unassigned_watcher_id(self): + def test_watch_unassigned_watcher_id(self): d = {} with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): self.watch(1, d) + def test_unwatch_non_dict(self): + with self.watcher() as wid: + with self.assertRaisesRegex(ValueError, r"Cannot watch non-dictionary"): + self.unwatch(wid, 1) + + def test_unwatch_out_of_range_watcher_id(self): + d = {} + with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"): + self.unwatch(-1, d) + with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID 8"): + self.unwatch(8, d) # DICT_MAX_WATCHERS = 8 + + def test_unwatch_unassigned_watcher_id(self): + d = {} + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): + self.unwatch(1, d) + + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"): + self.clear_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID 8"): + self.clear_watcher(8) # DICT_MAX_WATCHERS = 8 + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): + self.clear_watcher(1) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 28fb43dce4c6cb..173d7c2cb80530 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5296,6 +5296,20 @@ watch_dict(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +unwatch_dict(PyObject *self, PyObject *args) +{ + PyObject *dict; + int watcher_id; + if (!PyArg_ParseTuple(args, "iO", &watcher_id, &dict)) { + return NULL; + } + if (PyDict_Unwatch(watcher_id, dict)) { + return NULL; + } + Py_RETURN_NONE; +} + static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -5904,6 +5918,7 @@ static PyMethodDef TestMethods[] = { {"add_dict_watcher", add_dict_watcher, METH_O, NULL}, {"clear_dict_watcher", clear_dict_watcher, METH_O, NULL}, {"watch_dict", watch_dict, METH_VARARGS, NULL}, + {"unwatch_dict", unwatch_dict, METH_VARARGS, NULL}, {"get_dict_watcher_events", get_dict_watcher_events, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6542b1803ffa2e..97007479b1be91 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5720,6 +5720,20 @@ uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys) return v; } +static inline int +validate_watcher_id(PyInterpreterState *interp, int watcher_id) +{ + if (watcher_id < 0 || watcher_id >= DICT_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); + return -1; + } + if (!interp->dict_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); + return -1; + } + return 0; +} + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -5727,16 +5741,26 @@ PyDict_Watch(int watcher_id, PyObject* dict) PyErr_SetString(PyExc_ValueError, "Cannot watch non-dictionary"); return -1; } - if (watcher_id < 0 || watcher_id >= DICT_MAX_WATCHERS) { - PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (validate_watcher_id(interp, watcher_id)) { + return -1; + } + ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); + return 0; +} + +int +PyDict_Unwatch(int watcher_id, PyObject* dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_ValueError, "Cannot watch non-dictionary"); return -1; } PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!interp->dict_watchers[watcher_id]) { - PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); + if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); + ((PyDictObject*)dict)->ma_version_tag &= ~(1LL << watcher_id); return 0; } @@ -5759,13 +5783,8 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) int PyDict_ClearWatcher(int watcher_id) { - if (watcher_id < 0 || watcher_id >= DICT_MAX_WATCHERS) { - PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); - return -1; - } PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!interp->dict_watchers[watcher_id]) { - PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); + if (validate_watcher_id(interp, watcher_id)) { return -1; } interp->dict_watchers[watcher_id] = NULL; From 6b485629d2e3e232460db7da3f8b18b67d4f4da8 Mon Sep 17 00:00:00 2001 From: JasonYZ Date: Sat, 8 Oct 2022 05:46:23 +0100 Subject: [PATCH 04/10] gh-97822: Fix http.server documentation reference to test() function (#98027) Co-authored-by: Jelle Zijlstra --- Doc/library/http.server.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 48f952daae12f5..81b6bf5373b495 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -392,8 +392,8 @@ provides three different variants: contents of the file are output. If the file's MIME type starts with ``text/`` the file is opened in text mode; otherwise binary mode is used. - For example usage, see the implementation of the :func:`test` function - invocation in the :mod:`http.server` module. + For example usage, see the implementation of the ``test`` function + in :source:`Lib/http/server.py`. .. versionchanged:: 3.7 Support of the ``'If-Modified-Since'`` header. From 840fd19590935b91f9125bbf1cb93282d6073359 Mon Sep 17 00:00:00 2001 From: partev Date: Sat, 8 Oct 2022 01:55:35 -0400 Subject: [PATCH 05/10] [doc] Fix broken links to C extensions accelerating stdlib modules (#96914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa Co-authored-by: C.A.M. Gerlach --- Doc/license.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/license.rst b/Doc/license.rst index 00691b30ba6d3e..54643744dcf9c1 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -302,7 +302,8 @@ for third-party software incorporated in the Python distribution. Mersenne Twister ---------------- -The :mod:`_random` module includes code based on a download from +The :mod:`!_random` C extension underlying the :mod:`random` module +includes code based on a download from http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html. The following are the verbatim comments from the original code:: @@ -819,7 +820,8 @@ sources unless the build is configured ``--with-system-expat``:: libffi ------ -The :mod:`_ctypes` extension is built using an included copy of the libffi +The :mod:`!_ctypes` C extension underlying the :mod:`ctypes` module +is built using an included copy of the libffi sources unless the build is configured ``--with-system-libffi``:: Copyright (c) 1996-2008 Red Hat, Inc and others. @@ -920,7 +922,8 @@ on the cfuhash project:: libmpdec -------- -The :mod:`_decimal` module is built using an included copy of the libmpdec +The :mod:`!_decimal` C extension underlying the :mod:`decimal` module +is built using an included copy of the libmpdec library unless the build is configured ``--with-system-libmpdec``:: Copyright (c) 2008-2020 Stefan Krah. All rights reserved. From 296313002fde56f52d6c81f17d7ba5c2eb57d098 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 7 Oct 2022 23:54:16 -0700 Subject: [PATCH 06/10] gh-97913 Docs: Add walrus operator to the index (#97921) * Add walrus operator to the index * Add named expression to the index Co-authored-by: Mariatta Wijaya * Fix indentation and add missing newline Co-authored-by: Ezio Melotti Co-authored-by: Mariatta Wijaya Co-authored-by: Ezio Melotti --- Doc/reference/expressions.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 11f49a8c33dc88..28c17566009fbd 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1741,6 +1741,12 @@ returns a boolean value regardless of the type of its argument (for example, ``not 'foo'`` produces ``False`` rather than ``''``.) +.. index:: + single: := (colon equals) + single: assignment expression + single: walrus operator + single: named expression + Assignment expressions ====================== From 531ffaa7cdc58c5df2abe505803394dbd5293602 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 8 Oct 2022 09:11:38 +0200 Subject: [PATCH 07/10] Add `@ezio-melotti` as codeowner for `.github/`. (#98079) --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2568560c074f64..585589d6ce3bf7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,9 @@ # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format +# GitHub +.github/** @ezio-melotti + # asyncio **/*asyncio* @1st1 @asvetlov @gvanrossum From c66dbddfbaa374a6954897809574ee9fb463e393 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Oct 2022 11:13:34 +0300 Subject: [PATCH 08/10] GitHub Workflows security hardening (#96492) * Update project-updater.yml Signed-off-by: sashashura <93376818+sashashura@users.noreply.github.com> * Update project-updater.yml repository-projects: write is not needed because a separate secrets.ADD_TO_PROJECT_PAT is used Signed-off-by: sashashura <93376818+sashashura@users.noreply.github.com> --- .github/workflows/project-updater.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 77e55ed019b25a..99c7a05ae8cab0 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -6,6 +6,9 @@ on: - opened - labeled +permissions: + contents: read + jobs: add-to-project: name: Add issues to projects From 83eb827247dd28b13fd816936c74c162e9f52a2d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 8 Oct 2022 07:57:09 -0700 Subject: [PATCH 09/10] gh-97922: Run the GC only on eval breaker (#97920) --- Doc/whatsnew/3.12.rst | 7 +++++ Include/internal/pycore_gc.h | 2 ++ Include/internal/pycore_interp.h | 2 ++ Lib/test/test_frame.py | 2 +- ...2-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst | 5 ++++ Modules/gcmodule.c | 27 +++++++++++++++-- Modules/signalmodule.c | 13 ++++++++ Python/ceval_gil.c | 30 ++++++++++++------- 8 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f873974b3e78fe..341e85103a3cf7 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -93,6 +93,13 @@ Other Language Changes when parsing source code containing null bytes. (Contributed by Pablo Galindo in :gh:`96670`.) +* The Garbage Collector now runs only on the eval breaker mechanism of the + Python bytecode evaluation loop instead on object allocations. The GC can + also run when :c:func:`PyErr_CheckSignals` is called so C extensions that + need to run for a long time without executing any Python code also have a + chance to execute the GC periodically. (Contributed by Pablo Galindo in + :gh:`97922`.) + New Modules =========== diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index bfab0adfffc9ff..b3abe2030a03da 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -202,6 +202,8 @@ extern void _PyList_ClearFreeList(PyInterpreterState *interp); extern void _PyDict_ClearFreeList(PyInterpreterState *interp); extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); extern void _PyContext_ClearFreeList(PyInterpreterState *interp); +extern void _Py_ScheduleGC(PyInterpreterState *interp); +extern void _Py_RunGC(PyThreadState *tstate); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 8cecd5ab3e541e..c11e897305d42b 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -49,6 +49,8 @@ struct _ceval_state { _Py_atomic_int eval_breaker; /* Request for dropping the GIL */ _Py_atomic_int gil_drop_request; + /* The GC is ready to be executed */ + _Py_atomic_int gc_scheduled; struct _pending_calls pending; }; diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 4b86a60d2f4c36..4b5bb7f94ac469 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -277,7 +277,7 @@ def callback(phase, info): frame! """ nonlocal sneaky_frame_object - sneaky_frame_object = sys._getframe().f_back + sneaky_frame_object = sys._getframe().f_back.f_back # We're done here: gc.callbacks.remove(callback) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst new file mode 100644 index 00000000000000..bf78709362f464 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst @@ -0,0 +1,5 @@ +The Garbage Collector now runs only on the eval breaker mechanism of the +Python bytecode evaluation loop instead on object allocations. The GC can +also run when :c:func:`PyErr_CheckSignals` is called so C extensions that +need to run for a long time without executing any Python code also have a +chance to execute the GC periodically. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 97cb6e6e1efb1f..75832e9dd3da63 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2252,6 +2252,20 @@ PyObject_IS_GC(PyObject *obj) return _PyObject_IS_GC(obj); } +void +_Py_ScheduleGC(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (gcstate->collecting == 1) { + return; + } + struct _ceval_state *ceval = &interp->ceval; + if (!_Py_atomic_load_relaxed(&ceval->gc_scheduled)) { + _Py_atomic_store_relaxed(&ceval->gc_scheduled, 1); + _Py_atomic_store_relaxed(&ceval->eval_breaker, 1); + } +} + void _PyObject_GC_Link(PyObject *op) { @@ -2269,12 +2283,19 @@ _PyObject_GC_Link(PyObject *op) !gcstate->collecting && !_PyErr_Occurred(tstate)) { - gcstate->collecting = 1; - gc_collect_generations(tstate); - gcstate->collecting = 0; + _Py_ScheduleGC(tstate->interp); } } +void +_Py_RunGC(PyThreadState *tstate) +{ + GCState *gcstate = &tstate->interp->gc; + gcstate->collecting = 1; + gc_collect_generations(tstate); + gcstate->collecting = 0; +} + static PyObject * gc_alloc(size_t basicsize, size_t presize) { diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 0f30b4da036313..b85d6d19e8cd05 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1798,6 +1798,19 @@ int PyErr_CheckSignals(void) { PyThreadState *tstate = _PyThreadState_GET(); + + /* Opportunistically check if the GC is scheduled to run and run it + if we have a request. This is done here because native code needs + to call this API if is going to run for some time without executing + Python code to ensure signals are handled. Checking for the GC here + allows long running native code to clean cycles created using the C-API + even if it doesn't run the evaluation loop */ + struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; + if (_Py_atomic_load_relaxed(&interp_ceval_state->gc_scheduled)) { + _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); + _Py_RunGC(tstate); + } + if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index fd737b5738e889..9b9d7dc1d1af1e 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -5,6 +5,7 @@ #include "pycore_pyerrors.h" // _PyErr_Fetch() #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_interp.h" // _Py_RunGC() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() /* @@ -69,7 +70,8 @@ COMPUTE_EVAL_BREAKER(PyInterpreterState *interp, && _Py_ThreadCanHandleSignals(interp)) | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do) && _Py_ThreadCanHandlePendingCalls()) - | ceval2->pending.async_exc); + | ceval2->pending.async_exc + | _Py_atomic_load_relaxed_int32(&ceval2->gc_scheduled)); } @@ -938,6 +940,7 @@ _Py_HandlePending(PyThreadState *tstate) { _PyRuntimeState * const runtime = &_PyRuntime; struct _ceval_runtime_state *ceval = &runtime->ceval; + struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; /* Pending signals */ if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) { @@ -947,20 +950,26 @@ _Py_HandlePending(PyThreadState *tstate) } /* Pending calls */ - struct _ceval_state *ceval2 = &tstate->interp->ceval; - if (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) { + if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->pending.calls_to_do)) { if (make_pending_calls(tstate->interp) != 0) { return -1; } } + /* GC scheduled to run */ + if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gc_scheduled)) { + _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); + COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); + _Py_RunGC(tstate); + } + /* GIL drop request */ - if (_Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request)) { + if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { /* Give another thread a chance */ if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) { Py_FatalError("tstate mix-up"); } - drop_gil(ceval, ceval2, tstate); + drop_gil(ceval, interp_ceval_state, tstate); /* Other threads may run now */ @@ -981,16 +990,17 @@ _Py_HandlePending(PyThreadState *tstate) return -1; } -#ifdef MS_WINDOWS - // bpo-42296: On Windows, _PyEval_SignalReceived() can be called in a - // different thread than the Python thread, in which case + + // It is possible that some of the conditions that trigger the eval breaker + // are called in a different thread than the Python thread. An example of + // this is bpo-42296: On Windows, _PyEval_SignalReceived() can be called in + // a different thread than the Python thread, in which case // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the // current Python thread with the correct _Py_ThreadCanHandleSignals() // value. It prevents to interrupt the eval loop at every instruction if // the current Python thread cannot handle signals (if // _Py_ThreadCanHandleSignals() is false). - COMPUTE_EVAL_BREAKER(tstate->interp, ceval, ceval2); -#endif + COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); return 0; } From 4ed00be98f5b7dbac3ab71159dda907c931de486 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Sat, 8 Oct 2022 07:57:47 -0700 Subject: [PATCH 10/10] gh-68686: Retire eptag ptag scripts (#98064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Retire eptag ptag scripts * 📜🤖 Added by blurb_it. * fix news entry error Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- ...2-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst | 1 + Tools/scripts/README | 1 - Tools/scripts/eptags.py | 57 ------------------- Tools/scripts/ptags.py | 54 ------------------ 4 files changed, 1 insertion(+), 112 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst delete mode 100755 Tools/scripts/eptags.py delete mode 100755 Tools/scripts/ptags.py diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst new file mode 100644 index 00000000000000..a4289d675703b3 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst @@ -0,0 +1 @@ +Remove ptags and eptags scripts. diff --git a/Tools/scripts/README b/Tools/scripts/README index b53b0f21d7c293..70ea5f4cd0feae 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -5,7 +5,6 @@ useful while building, extending or managing Python. abitype.py Converts a C file to use the PEP 384 type definition API combinerefs.py A helper for analyzing PYTHONDUMPREFS output diff.py Print file diffs in context, unified, or ndiff formats -eptags.py Create Emacs TAGS file for Python modules gprof2html.py Transform gprof(1) output into useful HTML idle3 Main program to start IDLE md5sum.py Print MD5 checksums of argument files diff --git a/Tools/scripts/eptags.py b/Tools/scripts/eptags.py deleted file mode 100755 index 7f8059ba71adf3..00000000000000 --- a/Tools/scripts/eptags.py +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env python3 -"""Create a TAGS file for Python programs, usable with GNU Emacs. - -usage: eptags pyfiles... - -The output TAGS file is usable with Emacs version 18, 19, 20. -Tagged are: - - functions (even inside other defs or classes) - - classes - -eptags warns about files it cannot open. -eptags will not give warnings about duplicate tags. - -BUGS: - Because of tag duplication (methods with the same name in different - classes), TAGS files are not very useful for most object-oriented - python projects. -""" -import sys,re - -expr = r'^[ \t]*(def|class)[ \t]+([a-zA-Z_][a-zA-Z0-9_]*)[ \t]*[:\(]' -matcher = re.compile(expr) - -def treat_file(filename, outfp): - """Append tags found in file named 'filename' to the open file 'outfp'""" - try: - fp = open(filename, 'r') - except OSError: - sys.stderr.write('Cannot open %s\n'%filename) - return - with fp: - charno = 0 - lineno = 0 - tags = [] - size = 0 - while 1: - line = fp.readline() - if not line: - break - lineno = lineno + 1 - m = matcher.search(line) - if m: - tag = m.group(0) + '\177%d,%d\n' % (lineno, charno) - tags.append(tag) - size = size + len(tag) - charno = charno + len(line) - outfp.write('\f\n%s,%d\n' % (filename,size)) - for tag in tags: - outfp.write(tag) - -def main(): - with open('TAGS', 'w') as outfp: - for filename in sys.argv[1:]: - treat_file(filename, outfp) - -if __name__=="__main__": - main() diff --git a/Tools/scripts/ptags.py b/Tools/scripts/ptags.py deleted file mode 100755 index eedd411702c199..00000000000000 --- a/Tools/scripts/ptags.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/env python3 - -# ptags -# -# Create a tags file for Python programs, usable with vi. -# Tagged are: -# - functions (even inside other defs or classes) -# - classes -# - filenames -# Warns about files it cannot open. -# No warnings about duplicate tags. - -import sys, re, os - -tags = [] # Modified global variable! - -def main(): - args = sys.argv[1:] - for filename in args: - treat_file(filename) - if tags: - with open('tags', 'w') as fp: - tags.sort() - for s in tags: fp.write(s) - - -expr = r'^[ \t]*(def|class)[ \t]+([a-zA-Z0-9_]+)[ \t]*[:\(]' -matcher = re.compile(expr) - -def treat_file(filename): - try: - fp = open(filename, 'r') - except: - sys.stderr.write('Cannot open %s\n' % filename) - return - with fp: - base = os.path.basename(filename) - if base[-3:] == '.py': - base = base[:-3] - s = base + '\t' + filename + '\t' + '1\n' - tags.append(s) - while 1: - line = fp.readline() - if not line: - break - m = matcher.match(line) - if m: - content = m.group(0) - name = m.group(2) - s = name + '\t' + filename + '\t/^' + content + '/\n' - tags.append(s) - -if __name__ == '__main__': - main()