Merge remote-tracking branch 'upstream/main' into pythongh-111968-lis…

…t-fini

Doc/howto/descriptor.rst

@@ -1250,7 +1250,7 @@ instance::
     <function D.f at 0x00C45070>
 
     >>> d.f.__self__
-    <__main__.D object at 0x1012e1f98>
+    <__main__.D object at 0x00B18C90>
 
 If you have ever wondered where *self* comes from in regular methods or where
 *cls* comes from in class methods, this is it!

Doc/library/msvcrt.rst

@@ -75,10 +75,14 @@ File Operations
 .. function:: open_osfhandle(handle, flags)
 
    Create a C runtime file descriptor from the file handle *handle*. The *flags*
-   parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`,
-   and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter
+   parameter should be a bitwise OR of :const:`os.O_APPEND`,
+   :const:`os.O_RDONLY`, :const:`os.O_TEXT` and :const:`os.O_NOINHERIT`.
+   The returned file descriptor may be used as a parameter
    to :func:`os.fdopen` to create a file object.
 
+   The file descriptor is inheritable by default. Pass :const:`os.O_NOINHERIT`
+   flag to make it non inheritable.
+
    .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle
 
 

Include/internal/pycore_freelist.h

@@ -36,7 +36,7 @@ struct _Py_float_state {
 
 typedef struct _Py_freelist_state {
     struct _Py_float_state float_state;
-    struct _Py_list_state list;
+    struct _Py_list_state list_state;
 } _PyFreeListState;
 
 #ifdef __cplusplus

Include/internal/pycore_gc.h

@@ -122,6 +122,10 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) {
     PyGC_Head *gc = _Py_AS_GC(op);
     _PyGCHead_SET_FINALIZED(gc);
 }
+static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) {
+    PyGC_Head *gc = _Py_AS_GC(op);
+    gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED;
+}
 
 
 /* GC runtime state */

Lib/test/test_asyncgen.py

@@ -1701,6 +1701,14 @@ def test_asend(self):
         async def gen():
             yield 1
 
+        # gh-113753: asend objects allocated from a free-list should warn.
+        # Ensure there is a finalized 'asend' object ready to be reused.
+        try:
+            g = gen()
+            g.asend(None).send(None)
+        except StopIteration:
+            pass
+
         msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited"
         with self.assertWarnsRegex(RuntimeWarning, msg):
             g = gen()

Lib/test/test_builtin.py

@@ -629,8 +629,8 @@ def __dir__(self):
 
     def test___ne__(self):
         self.assertFalse(None.__ne__(None))
-        self.assertTrue(None.__ne__(0))
-        self.assertTrue(None.__ne__("abc"))
+        self.assertIs(None.__ne__(0), NotImplemented)
+        self.assertIs(None.__ne__("abc"), NotImplemented)
 
     def test_divmod(self):
         self.assertEqual(divmod(12, 7), (1, 5))

Lib/test/test_os.py

@@ -4485,6 +4485,61 @@ def test_openpty(self):
         self.assertEqual(os.get_inheritable(master_fd), False)
         self.assertEqual(os.get_inheritable(slave_fd), False)
 
+    @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.openpty()")
+    def test_pipe_spawnl(self):
+        # gh-77046: On Windows, os.pipe() file descriptors must be created with
+        # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to
+        # get (_osfile(fd) & _O_NOINHERIT), so use a functional test.
+        #
+        # Make sure that fd is not inherited by a child process created by
+        # os.spawnl(): get_osfhandle() and dup() must fail with EBADF.
+
+        fd, fd2 = os.pipe()
+        self.addCleanup(os.close, fd)
+        self.addCleanup(os.close, fd2)
+
+        code = textwrap.dedent(f"""
+            import errno
+            import os
+            import test.support
+            try:
+                import msvcrt
+            except ImportError:
+                msvcrt = None
+
+            fd = {fd}
+
+            with test.support.SuppressCrashReport():
+                if msvcrt is not None:
+                    try:
+                        handle = msvcrt.get_osfhandle(fd)
+                    except OSError as exc:
+                        if exc.errno != errno.EBADF:
+                            raise
+                        # get_osfhandle(fd) failed with EBADF as expected
+                    else:
+                        raise Exception("get_osfhandle() must fail")
+
+                try:
+                    fd3 = os.dup(fd)
+                except OSError as exc:
+                    if exc.errno != errno.EBADF:
+                        raise
+                    # os.dup(fd) failed with EBADF as expected
+                else:
+                    os.close(fd3)
+                    raise Exception("dup must fail")
+        """)
+
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, os_helper.TESTFN)
+        with open(filename, "w") as fp:
+            print(code, file=fp, end="")
+
+        cmd = [sys.executable, filename]
+        exitcode = os.spawnl(os.P_WAIT, cmd[0], *cmd)
+        self.assertEqual(exitcode, 0)
+
 
 class PathTConverterTests(unittest.TestCase):
     # tuples of (function name, allows fd arguments, additional arguments to

Lib/test/test_subprocess.py

@@ -835,16 +835,25 @@ def is_env_var_to_ignore(n):
                                if not is_env_var_to_ignore(k)]
             self.assertEqual(child_env_names, [])
 
+    @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1,
+                     'The Python shared library cannot be loaded '
+                     'without some system environments.')
+    @unittest.skipIf(check_sanitizer(address=True),
+                     'AddressSanitizer adds to the environment.')
     def test_one_environment_variable(self):
         newenv = {'fruit': 'orange'}
         cmd = [sys.executable, '-c',
                                'import sys,os;'
                                'sys.stdout.write("fruit="+os.getenv("fruit"))']
         if sys.platform == "win32":
             cmd = ["CMD", "/c", "SET", "fruit"]
-        with subprocess.Popen(cmd, stdout=subprocess.PIPE, env=newenv) as p:
-            stdout, _ = p.communicate()
-            self.assertTrue(stdout.startswith(b"fruit=orange"))
+        with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=newenv) as p:
+            stdout, stderr = p.communicate()
+            if p.returncode and support.verbose:
+                print("STDOUT:", stdout.decode("ascii", "replace"))
+                print("STDERR:", stderr.decode("ascii", "replace"))
+            self.assertEqual(p.returncode, 0)
+            self.assertEqual(stdout.strip(), b"fruit=orange")
 
     def test_invalid_cmd(self):
         # null character in the command name

Lib/test/test_zipfile/test_core.py

@@ -2272,6 +2272,7 @@ def test_decompress_without_3rd_party_library(self):
             with zipfile.ZipFile(zip_file) as zf:
                 self.assertRaises(RuntimeError, zf.extract, 'a.txt')
 
+    @requires_zlib()
     def test_full_overlap(self):
         data = (
             b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
@@ -2300,6 +2301,7 @@ def test_full_overlap(self):
             with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'):
                 zipf.read('b')
 
+    @requires_zlib()
     def test_quoted_overlap(self):
         data = (
             b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc'

Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst

@@ -0,0 +1,2 @@
+Fix an issue where the finalizer of ``PyAsyncGenASend`` objects might not be
+called if they were allocated from a free list.

Misc/NEWS.d/next/Core and Builtins/2024-01-08-14-34-02.gh-issue-77046.sDUh2d.rst

@@ -0,0 +1,3 @@
+On Windows, file descriptors wrapping Windows handles are now created non
+inheritable by default (:pep:`446`). Patch by Zackery Spytz and Victor
+Stinner.

Modules/_io/winconsoleio.c

@@ -391,9 +391,9 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
         }
 
         if (self->writable)
-            self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY);
+            self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY | _O_NOINHERIT);
         else
-            self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY);
+            self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY | _O_NOINHERIT);
         if (self->fd < 0) {
             PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj);
             CloseHandle(handle);

Modules/posixmodule.c

@@ -11578,8 +11578,8 @@ os_pipe_impl(PyObject *module)
     Py_BEGIN_ALLOW_THREADS
     ok = CreatePipe(&read, &write, &attr, 0);
     if (ok) {
-        fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY);
-        fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY);
+        fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY | _O_NOINHERIT);
+        fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY | _O_NOINHERIT);
         if (fds[0] == -1 || fds[1] == -1) {
             CloseHandle(read);
             CloseHandle(write);

Objects/genobject.c

@@ -6,6 +6,7 @@
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_EvalFrame()
 #include "pycore_frame.h"         // _PyInterpreterFrame
+#include "pycore_gc.h"            // _PyGC_CLEAR_FINALIZED()
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_modsupport.h"    // _PyArg_CheckPositional()
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
@@ -1739,6 +1740,7 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
 #endif
     if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) {
         assert(PyAsyncGenASend_CheckExact(o));
+        _PyGC_CLEAR_FINALIZED((PyObject *)o);
         state->asend_freelist[state->asend_numfree++] = o;
     }
     else

Objects/listobject.c

@@ -26,7 +26,7 @@ get_list_state(void)
 {
     _PyFreeListState *state = _PyFreeListState_GET();
     assert(state != NULL);
-    return &state->list;
+    return &state->list_state;
 }
 #endif
 
@@ -124,7 +124,7 @@ void
 _PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
 {
 #ifdef WITH_FREELISTS
-    struct _Py_list_state *state = &freelist_state->list;
+    struct _Py_list_state *state = &freelist_state->list_state;
     while (state->numfree > 0) {
         PyListObject *op = state->free_list[--state->numfree];
         assert(PyList_CheckExact(op));

Tools/build/generate_sbom.py

@@ -82,6 +82,14 @@ def spdx_id(value: str) -> str:
     return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value)
 
 
+def error_if(value: bool, error_message: str) -> None:
+    """Prints an error if a comparison fails along with a link to the devguide"""
+    if value:
+        print(error_message)
+        print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.")
+        sys.exit(1)
+
+
 def filter_gitignored_paths(paths: list[str]) -> list[str]:
     """
     Filter out paths excluded by the gitignore file.
@@ -206,22 +214,47 @@ def main() -> None:
     discover_pip_sbom_package(sbom_data)
 
     # Ensure all packages in this tool are represented also in the SBOM file.
-    assert {package["name"] for package in sbom_data["packages"]} == set(PACKAGE_TO_FILES)
+    error_if(
+        {package["name"] for package in sbom_data["packages"]} != set(PACKAGE_TO_FILES),
+        "Packages defined in SBOM tool don't match those defined in SBOM file.",
+    )
 
     # Make a bunch of assertions about the SBOM data to ensure it's consistent.
     for package in sbom_data["packages"]:
-
         # Properties and ID must be properly formed.
-        assert set(package.keys()) == REQUIRED_PROPERTIES_PACKAGE
-        assert package["SPDXID"] == spdx_id(f"SPDXRef-PACKAGE-{package['name']}")
+        error_if(
+            "name" not in package,
+            "Package is missing the 'name' field"
+        )
+        error_if(
+            set(package.keys()) != REQUIRED_PROPERTIES_PACKAGE,
+            f"Package '{package['name']}' is missing required fields",
+        )
+        error_if(
+            package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"),
+            f"Package '{package['name']}' has a malformed SPDXID",
+        )
 
         # Version must be in the download and external references.
         version = package["versionInfo"]
-        assert version in package["downloadLocation"]
-        assert all(version in ref["referenceLocator"] for ref in package["externalRefs"])
+        error_if(
+            version not in package["downloadLocation"],
+            f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field",
+        )
+        error_if(
+            any(version not in ref["referenceLocator"] for ref in package["externalRefs"]),
+            (
+                f"Version '{version}' for package '{package['name']} not in "
+                f"all 'externalRefs[].referenceLocator' fields"
+            ),
+        )
 
         # License must be on the approved list for SPDX.
-        assert package["licenseConcluded"] in ALLOWED_LICENSE_EXPRESSIONS, package["licenseConcluded"]
+        license_concluded = package["licenseConcluded"]
+        error_if(
+            license_concluded not in ALLOWED_LICENSE_EXPRESSIONS,
+            f"License identifier '{license_concluded}' not in SBOM tool allowlist"
+        )
 
     # Regenerate file information from current data.
     sbom_files = []
@@ -232,11 +265,13 @@ def main() -> None:
         package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
         exclude = files.exclude or ()
         for include in sorted(files.include):
-
             # Find all the paths and then filter them through .gitignore.
             paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True)
             paths = filter_gitignored_paths(paths)
-            assert paths, include  # Make sure that every value returns something!
+            error_if(
+                len(paths) == 0,
+                f"No valid paths found at path '{include}' for package '{name}",
+            )
 
             for path in paths:
                 # Skip directories and excluded files