Merge branch '3.8' into backport-77e0919-3.8

.github/workflows/build.yml

@@ -18,7 +18,7 @@ jobs:
       run_tests: ${{ steps.check.outputs.run_tests }}
       run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Check for source changes
         id: check
         run: |
@@ -50,8 +50,8 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
       - name: Install Dependencies
         run: |
             sudo ./.github/workflows/posix-deps-apt.sh
@@ -72,10 +72,23 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Check Autoconf version 2.69 and aclocal 1.16.3
+        run: |
+          grep "Generated by GNU Autoconf 2.69" configure
+          grep "aclocal 1.16.3" aclocal.m4
+          grep -q "runstatedir" configure
+          grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+      - name: Configure CPython
+        run: |
+          # Build Python with the libpython dynamic library
+          ./configure --config-cache --with-pydebug --enable-shared
+      - name: Regenerate autoconf files
+        run: |
+          make regen-configure
       - name: Build CPython
         run: |
           ./configure --with-pydebug
@@ -99,7 +112,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build CPython
       run: .\PCbuild\build.bat -e -p Win32
     - name: Display build info
@@ -113,7 +126,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build CPython
       run: .\PCbuild\build.bat -e -p x64
     - name: Display build info
@@ -131,7 +144,7 @@ jobs:
       HOMEBREW_NO_AUTO_UPDATE: 1
       HOMEBREW_NO_INSTALL_CLEANUP: 1
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Homebrew dependencies
       run: brew install pkg-config [email protected] xz gdbm tcl-tk
     - name: Configure CPython
@@ -160,7 +173,7 @@ jobs:
     env:
       OPENSSL_VER: 1.1.1v
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars
@@ -206,7 +219,7 @@ jobs:
       OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars

.github/workflows/build_msi.yml

@@ -23,14 +23,14 @@ jobs:
     name: 'Windows (x86) Installer'
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build CPython installer
       run: .\Tools\msi\build.bat -x86
 
   build_win_amd64:
     name: 'Windows (x64) Installer'
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build CPython installer
       run: .\Tools\msi\build.bat -x64

.github/workflows/coverage.yml

@@ -25,7 +25,7 @@ jobs:
     env:
       OPENSSL_VER: 1.1.1n
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: 'Restore OpenSSL build'
@@ -73,7 +73,7 @@ jobs:
     name: 'Ubuntu (C Coverage)'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure CPython

.github/workflows/doc.yml

@@ -22,7 +22,7 @@ jobs:
     name: 'Docs'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: 'Install Dependencies'
       run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican
     - name: 'Configure CPython'

.github/workflows/verify-ensurepip-wheels.yml

@@ -20,7 +20,7 @@ jobs:
   verify:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3'

.gitignore

@@ -120,3 +120,11 @@ Tools/ssl/win32
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python
 !/Python/
+
+# Artifacts generated by 3.11 lying around when switching branches:
+/_bootstrap_python
+/Programs/_freeze_module
+/Modules/Setup.bootstrap
+/Modules/Setup.stdlib
+/Python/deepfreeze/
+/Python/frozen_modules/ 

.readthedocs.yml

@@ -0,0 +1,12 @@
+# This is a dummy config file so that readthedocs.org doesn't fail on security branches.
+# Note that this won't result in docs actually getting built;
+# clicking on the docs preview link on a PR will result in a 404.
+version: 2
+formats: []
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
+  jobs:
+    post_checkout:
+    - exit 183

Doc/constraints.txt

@@ -0,0 +1,24 @@
+# We have upper bounds on our transitive dependencies here
+# To avoid new releases unexpectedly breaking our build.
+# This file can be updated on an ad-hoc basis,
+# though it will probably have to be updated
+# whenever Doc/requirements.txt is updated.
+
+# Direct dependencies of Sphinx
+babel<3
+colorama<0.5
+imagesize<1.5
+Jinja2<3.2
+packaging<24
+Pygments>=2.16.1,<3
+requests<3
+snowballstemmer<3
+sphinxcontrib-applehelp<1.0.5
+sphinxcontrib-devhelp<1.0.6
+sphinxcontrib-htmlhelp<2.0.5
+sphinxcontrib-jsmath<1.1
+sphinxcontrib-qthelp<1.0.7
+sphinxcontrib-serializinghtml<1.1.10
+
+# Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above)
+MarkupSafe<2.2

Doc/requirements.txt

@@ -10,9 +10,13 @@ sphinx==2.4.4
 docutils==0.17.1
 # Jinja version is pinned to a version compatible with Sphinx version 2.4.4.
 jinja2==3.0.3
+# Alabaster version is pinned to a version compatible with Sphinx version 2.4.4.
+alabaster==0.7.13
 
 blurb
 
 # The theme used by the documentation is stored separately, so we need
 # to install that as well.
 python-docs-theme
+
+-c constraints.txt

Include/patchlevel.h

@@ -18,12 +18,12 @@
 /*--start constants--*/
 #define PY_MAJOR_VERSION        3
 #define PY_MINOR_VERSION        8
-#define PY_MICRO_VERSION        17
+#define PY_MICRO_VERSION        18
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL       0
 
 /* Version as a string */
-#define PY_VERSION              "3.8.17+"
+#define PY_VERSION              "3.8.18+"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.

Lib/site.py

@@ -74,6 +74,7 @@
 import builtins
 import _sitebuiltins
 import io
+import stat
 
 # Prefixes for site-packages; add additional prefixes like /usr/local here
 PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -156,6 +157,13 @@ def addpackage(sitedir, name, known_paths):
     else:
         reset = False
     fullname = os.path.join(sitedir, name)
+    try:
+        st = os.lstat(fullname)
+    except OSError:
+        return
+    if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or
+        (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)):
+        return
     try:
         f = io.TextIOWrapper(io.open_code(fullname))
     except OSError:
@@ -203,7 +211,8 @@ def addsitedir(sitedir, known_paths=None):
         names = os.listdir(sitedir)
     except OSError:
         return
-    names = [name for name in names if name.endswith(".pth")]
+    names = [name for name in names
+             if name.endswith(".pth") and not name.startswith(".")]
     for name in sorted(names):
         addpackage(sitedir, name, known_paths)
     if reset:

Lib/ssl.py

@@ -1048,7 +1048,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True,
                     self.close()
                 except OSError:
                     pass
-                raise notconn_pre_handshake_data_error
+                try:
+                    raise notconn_pre_handshake_data_error
+                finally:
+                    # Explicitly break the reference cycle.
+                    notconn_pre_handshake_data_error = None
         else:
             connected = True
 

Lib/tempfile.py

@@ -263,6 +263,22 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type):
     raise FileExistsError(_errno.EEXIST,
                           "No usable temporary file name found")
 
+def _dont_follow_symlinks(func, path, *args):
+    # Pass follow_symlinks=False, unless not supported on this platform.
+    if func in _os.supports_follow_symlinks:
+        func(path, *args, follow_symlinks=False)
+    elif _os.name == 'nt' or not _os.path.islink(path):
+        func(path, *args)
+
+def _resetperms(path):
+    try:
+        chflags = _os.chflags
+    except AttributeError:
+        pass
+    else:
+        _dont_follow_symlinks(chflags, path, 0)
+    _dont_follow_symlinks(_os.chmod, path, 0o700)
+
 
 # User visible interfaces.
 
@@ -786,17 +802,10 @@ def __init__(self, suffix=None, prefix=None, dir=None):
     def _rmtree(cls, name):
         def onerror(func, path, exc_info):
             if issubclass(exc_info[0], PermissionError):
-                def resetperms(path):
-                    try:
-                        _os.chflags(path, 0)
-                    except AttributeError:
-                        pass
-                    _os.chmod(path, 0o700)
-
                 try:
                     if path != name:
-                        resetperms(_os.path.dirname(path))
-                    resetperms(path)
+                        _resetperms(_os.path.dirname(path))
+                    _resetperms(path)
 
                     try:
                         _os.unlink(path)

Lib/test/test_codecencodings_iso2022.py

@@ -24,6 +24,52 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase):
         (b'ab\x1BNdef', 'replace', 'abdef'),
     )
 
+class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'iso2022_jp_3'
+    tstring = multibytecodec_support.load_teststring('iso2022_jp')
+    codectests = COMMON_CODEC_TESTS + (
+        (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
+        (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402'      ),
+        (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B'  ),
+        (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'),
+        (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02'      ),
+        (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2'  ),
+        ('\u3402',       'strict', b'\x1B$(O\x2E\x23\x1B(B'),
+        ('\U0002000B',   'strict', b'\x1B$(O\x2E\x22\x1B(B'),
+        ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'),
+        ('\u4E02',       'strict', b'\x1B$(P\x21\x22\x1B(B'),
+        ('\U0002A6B2',   'strict', b'\x1B$(P\x7E\x76\x1B(B'),
+        (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'),
+        ('ab\u4FF1def', 'replace', b'ab?def'),
+    )
+    xmlcharnametest = (
+        '\xAB\u211C\xBB = \u2329\u1234\u232A',
+        b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩'
+    )
+
+class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'iso2022_jp_2004'
+    tstring = multibytecodec_support.load_teststring('iso2022_jp')
+    codectests = COMMON_CODEC_TESTS + (
+        (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
+        (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402'      ),
+        (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B'  ),
+        (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'),
+        (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02'      ),
+        (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2'  ),
+        ('\u3402',       'strict', b'\x1B$(Q\x2E\x23\x1B(B'),
+        ('\U0002000B',   'strict', b'\x1B$(Q\x2E\x22\x1B(B'),
+        ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'),
+        ('\u4E02',       'strict', b'\x1B$(P\x21\x22\x1B(B'),
+        ('\U0002A6B2',   'strict', b'\x1B$(P\x7E\x76\x1B(B'),
+        (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'),
+        ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'),
+    )
+    xmlcharnametest = (
+        '\xAB\u211C\xBB = \u2329\u1234\u232A',
+        b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩'
+    )
+
 class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'iso2022_kr'
     tstring = multibytecodec_support.load_teststring('iso2022_kr')

Lib/test/test_httpservers.py

@@ -417,10 +417,10 @@ def test_undecodable_filename(self):
     def test_undecodable_parameter(self):
         # sanity check using a valid parameter
         response = self.request(self.base_url + '/?x=123').read()
-        self.assertRegex(response, f'listing for {self.base_url}/\?x=123'.encode('latin1'))
+        self.assertRegex(response, f'listing for {self.base_url}/\\?x=123'.encode('latin1'))
         # now the bogus encoding
         response = self.request(self.base_url + '/?x=%bb').read()
-        self.assertRegex(response, f'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1'))
+        self.assertRegex(response, f'listing for {self.base_url}/\\?x=\xef\xbf\xbd'.encode('latin1'))
 
     def test_get_dir_redirect_location_domain_injection_bug(self):
         """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location.

Lib/test/test_mmap.py

@@ -240,10 +240,15 @@ def test_access_parameter(self):
             # Try writing with PROT_EXEC and without PROT_WRITE
             prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0)
             with open(TESTFN, "r+b") as f:
-                m = mmap.mmap(f.fileno(), mapsize, prot=prot)
-                self.assertRaises(TypeError, m.write, b"abcdef")
-                self.assertRaises(TypeError, m.write_byte, 0)
-                m.close()
+                try:
+                    m = mmap.mmap(f.fileno(), mapsize, prot=prot)
+                except PermissionError:
+                    # on macOS 14, PROT_READ | PROT_WRITE is not allowed
+                    pass
+                else:
+                    self.assertRaises(TypeError, m.write, b"abcdef")
+                    self.assertRaises(TypeError, m.write_byte, 0)
+                    m.close()
 
     def test_bad_file_desc(self):
         # Try opening a bad file descriptor...