Merge branch 'main' into supermon

* main: pythongh-91896: Fixup some docs issues following ByteString deprecation (python#104422) pythonGH-104371: check return value of calling `mv.release` (python#104417) pythongh-104415: Fix refleak tests for `typing.ByteString` deprecation (python#104416) pythonGH-86275: Implementation of hypothesis stubs for property-based tests, with zoneinfo tests (python#22863) pythonGH-103082: Filter LINE events in VM, to simplify tool implementation. (pythonGH-104387) pythongh-93649: Split gc- and allocation tests from _testcapimodule.c (pythonGH-104403) pythongh-104389: Add 'unused' keyword to Argument Clinic C converters (python#104390) pythongh-101819: Prepare _io._IOBase for module state (python#104386) pythongh-104413: Fix refleak when super attribute throws AttributeError (python#104414) Fix refleak in `super_descr_get` (python#104408) pythongh-87526: Remove dead initialization from _zoneinfo parse_abbr() (python#24700) pythongh-91896: Improve visibility of `ByteString` deprecation warnings (python#104294) pythongh-104371: Fix calls to `__release_buffer__` while an exception is active (python#104378) pythongh-104377: fix cell in comprehension that is free in outer scope (python#104394) pythongh-104392: Remove _paramspec_tvars from typing (python#104393) pythongh-104396: uuid.py to skip platform check for emscripten and wasi (pythongh-104397) pythongh-99108: Refresh HACL* from upstream (python#104401) pythongh-104301: Allow leading whitespace in disambiguated pdb statements (python#104342)
carljm · May 12, 2023 · 8efd7c6 · 8efd7c6
2 parents fb47955 + ce4eecf
commit 8efd7c6
Show file tree

Hide file tree

Showing 60 changed files with 1,786 additions and 654 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -36,6 +36,7 @@ jobs:
     timeout-minutes: 10
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -61,6 +62,17 @@ jobs:
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
+          # Check if we should run hypothesis tests
+          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+          echo $GIT_BRANCH
+          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+            echo "Branch too old for hypothesis tests"
+            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+          else
+            echo "Run hypothesis tests"
+            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+          fi
+
   check_generated_files:
     name: 'Check if generated files are up to date'
     runs-on: ubuntu-latest
@@ -291,6 +303,90 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  test_hypothesis:
+    name: "Hypothesis Tests on Ubuntu"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    env:
+      OPENSSL_VER: 1.1.1t
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v3
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: "Create hypothesis venv"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        VENV_LOC=$(realpath -m .)/hypovenv
+        VENV_PYTHON=$VENV_LOC/bin/python
+        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: "Run tests"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        # Most of the excluded tests are slow test suites with no property tests
+        #
+        # (GH-104097) test_sysconfig is skipped because it has tests that are
+        # failing when executed from inside a virtual environment.
+        ${{ env.VENV_PYTHON }} -m test \
+          -W \
+          -x test_asyncio \
+          -x test_multiprocessing_fork \
+          -x test_multiprocessing_forkserver \
+          -x test_multiprocessing_spawn \
+          -x test_concurrent_futures \
+          -x test_socket \
+          -x test_subprocess \
+          -x test_signal \
+          -x test_sysconfig
+
 
   build_asan:
     name: 'Address sanitizer'

diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
@@ -775,6 +775,9 @@ All Argument Clinic converters accept the following arguments:
     because :pep:`8` mandates that the Python library may not use
     annotations.
 
+  ``unused``
+    Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature.
+
 In addition, some converters accept additional arguments.  Here is a list
 of these arguments, along with their meanings:
 

diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
@@ -14,7 +14,10 @@
 
 .. testsetup:: *
 
-   from collections.abc import *
+   import warnings
+   # Ignore warning when ByteString is imported
+   with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
+       from collections.abc import *
    import itertools
    __name__ = '<doctest>'
 

diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst
@@ -602,9 +602,17 @@ can be overridden by the local file.
 
    Execute the (one-line) *statement* in the context of the current stack frame.
    The exclamation point can be omitted unless the first word of the statement
-   resembles a debugger command.  To set a global variable, you can prefix the
-   assignment command with a :keyword:`global` statement on the same line,
-   e.g.::
+   resembles a debugger command, e.g.:
+
+   .. code-block:: none
+
+      (Pdb) ! n=42
+      (Pdb)
+
+   To set a global variable, you can prefix the assignment command with a
+   :keyword:`global` statement on the same line, e.g.:
+
+   .. code-block:: none
 
       (Pdb) global list_options; list_options = ['-l']
       (Pdb)

diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
@@ -831,6 +831,9 @@ Pending Removal in Python 3.14
   For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`.
   (Contributed by Shantanu Jain in :gh:`91896`.)
 
+* :class:`typing.ByteString`, deprecated since Python 3.9, now causes an
+  :exc:`DeprecationWarning` to be emitted when it is used or accessed.
+
 * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable
   bases using the C API.
 

diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
@@ -19,7 +19,6 @@ struct _frame {
     struct _PyInterpreterFrame *f_frame; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
-    int f_last_traced_line;     /* The last line traced for this frame */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
     char f_fast_as_locals;      /* Have the fast locals of this frame been converted to a dict? */

diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h
@@ -69,13 +69,13 @@ _Py_call_instrumentation(PyThreadState *tstate, int event,
 
 extern int
 _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
-                              _Py_CODEUNIT *instr);
+                              _Py_CODEUNIT *instr, _Py_CODEUNIT *prev);
 
 extern int
 _Py_call_instrumentation_instruction(
     PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr);
 
-int
+_Py_CODEUNIT *
 _Py_call_instrumentation_jump(
     PyThreadState *tstate, int event,
     _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target);
@@ -100,6 +100,7 @@ extern int
 _Py_Instrumentation_GetLine(PyCodeObject *code, int index);
 
 extern PyObject _PyInstrumentation_MISSING;
+extern PyObject _PyInstrumentation_DISABLE;
 
 #ifdef __cplusplus
 }

diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py
@@ -1,3 +1,12 @@
 from _collections_abc import *
 from _collections_abc import __all__
 from _collections_abc import _CallableGenericAlias
+
+_deprecated_ByteString = globals().pop("ByteString")
+
+def __getattr__(attr):
+    if attr == "ByteString":
+        import warnings
+        warnings._deprecated("collections.abc.ByteString", remove=(3, 14))
+        return _deprecated_ByteString
+    raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}")
diff --git a/Lib/pdb.py b/Lib/pdb.py
@@ -440,7 +440,7 @@ def displayhook(self, obj):
             self.message(repr(obj))
 
     def default(self, line):
-        if line[:1] == '!': line = line[1:]
+        if line[:1] == '!': line = line[1:].strip()
         locals = self.curframe_locals
         globals = self.curframe.f_globals
         try:
@@ -1642,9 +1642,12 @@ def help_exec(self):
 
         Execute the (one-line) statement in the context of the current
         stack frame.  The exclamation point can be omitted unless the
-        first word of the statement resembles a debugger command.  To
-        assign to a global variable you must always prefix the command
-        with a 'global' command, e.g.:
+        first word of the statement resembles a debugger command, e.g.:
+        (Pdb) ! n=42
+        (Pdb)
+
+        To assign to a global variable you must always prefix the command with
+        a 'global' command, e.g.:
         (Pdb) global list_options; list_options = ['-l']
         (Pdb)
         """

diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py
@@ -5283,11 +5283,14 @@
              'current\n'
              '   stack frame. The exclamation point can be omitted unless the '
              'first\n'
-             '   word of the statement resembles a debugger command.  To set '
-             'a\n'
-             '   global variable, you can prefix the assignment command with '
-             'a\n'
-             '   "global" statement on the same line, e.g.:\n'
+             '   word of the statement resembles a debugger command, e.g.:'
+             '\n'
+             '      (Pdb) ! n=42\n'
+             '      (Pdb)\n'
+             '\n'
+             '   To set a global variable, you can prefix the assignment command '
+             '   with \n'
+             '   a "global" statement on the same line, e.g.:\n'
              '\n'
              "      (Pdb) global list_options; list_options = ['-l']\n"
              '      (Pdb)\n'

diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
@@ -48,11 +48,13 @@ def dash_R(ns, test_name, test_func):
     else:
         zdc = zipimport._zip_directory_cache.copy()
     abcs = {}
-    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
-        if not isabstract(abc):
-            continue
-        for obj in abc.__subclasses__() + [abc]:
-            abcs[obj] = _get_dump(obj)[0]
+    # catch and ignore collections.abc.ByteString deprecation
+    with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
+        for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
+            if not isabstract(abc):
+                continue
+            for obj in abc.__subclasses__() + [abc]:
+                abcs[obj] = _get_dump(obj)[0]
 
     # bpo-31217: Integer pool to get a single integer object for the same
     # value. The pool is used to prevent false alarm when checking for memory
@@ -173,7 +175,9 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
         zipimport._zip_directory_cache.update(zdc)
 
     # Clear ABC registries, restoring previously saved ABC registries.
-    abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
+    # ignore deprecation warning for collections.abc.ByteString
+    with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
+        abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
     abs_classes = filter(isabstract, abs_classes)
     for abc in abs_classes:
         for obj in abc.__subclasses__() + [abc]:

diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py
@@ -257,8 +257,10 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved):
         sysconfig._INSTALL_SCHEMES.update(saved[2])
 
     def get_files(self):
+        # XXX: Maybe add an allow-list here?
         return sorted(fn + ('/' if os.path.isdir(fn) else '')
-                      for fn in os.listdir())
+                      for fn in os.listdir()
+                      if not fn.startswith(".hypothesis"))
     def restore_files(self, saved_value):
         fn = os_helper.TESTFN
         if fn not in saved_value and (fn + '/') not in saved_value: