From 822db860eada721742f878653d7ac9364ed8df59 Mon Sep 17 00:00:00 2001 From: Md Sadman Chowdhury <61442059+SpicyCatGames@users.noreply.github.com> Date: Sun, 2 Jul 2023 01:47:14 +0600 Subject: [PATCH 01/31] Fix duplicate word typos in comments (#106225) --- Include/cpython/object.h | 2 +- Objects/frameobject.c | 2 +- Python/initconfig.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 71d268fae1a6dc..d681435c845459 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -231,7 +231,7 @@ struct _typeobject { }; /* This struct is used by the specializer - * It should should be treated as an opaque blob + * It should be treated as an opaque blob * by code other than the specializer and interpreter. */ struct _specialization_cache { // In order to avoid bloating the bytecode with lots of inline caches, the diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 98f4a0378bf76d..0158d72d6b4f98 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1186,7 +1186,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, // (likely) MAKE_CELL must have executed already. value = PyCell_GET(value); } - // (likely) Otherwise it it is an arg (kind & CO_FAST_LOCAL), + // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), // with the initial value set when the frame was created... // (unlikely) ...or it was set to some initial value by // an earlier call to PyFrame_LocalsToFast(). diff --git a/Python/initconfig.c b/Python/initconfig.c index 1dcefd478eefea..147cb370412cd6 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -538,7 +538,7 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); // XXX _PyRuntime.orig_argv only gets cleared by Py_Main(), - // so it it currently leaks for embedders. + // so it currently leaks for embedders. res = _PyWideStringList_Copy(&_PyRuntime.orig_argv, &argv_list); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); From 46d77610fc77088bceac720a13d9f2df3a50f29e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 00:27:18 +0200 Subject: [PATCH 02/31] gh-106316: Remove pytime.h header file (#106317) Remove the "cpython/pytime.h" header file: it only contained private functions. Move functions to the internal pycore_time.h header file. Move tests from _testcapi to _testinternalcapi. Rename also test methods to have the same name than tested C functions. No longer export these functions: * _PyTime_Add() * _PyTime_As100Nanoseconds() * _PyTime_FromMicrosecondsClamp() * _PyTime_FromTimespec() * _PyTime_FromTimeval() * _PyTime_GetPerfCounterWithInfo() * _PyTime_MulDiv() --- .github/CODEOWNERS | 2 +- Doc/whatsnew/3.13.rst | 3 + Include/Python.h | 1 - Include/cpython/pytime.h | 331 ------------------ Include/internal/pycore_import.h | 2 + Include/internal/pycore_time.h | 312 +++++++++++++++++ Lib/test/test_time.py | 89 ++--- Makefile.pre.in | 1 - ...-07-01-21-23-33.gh-issue-106316.hp2Ijw.rst | 2 + Modules/Setup.stdlib.in | 2 +- Modules/_queuemodule.c | 1 + Modules/_testcapi/parts.h | 1 - Modules/_testcapi/pytime.c | 274 --------------- Modules/_testcapimodule.c | 3 - Modules/_testinternalcapi.c | 266 ++++++++++++++ Modules/_testsinglephase.c | 1 + Modules/selectmodule.c | 1 + Modules/socketmodule.h | 2 + PCbuild/_testcapi.vcxproj | 3 +- PCbuild/_testcapi.vcxproj.filters | 5 +- PCbuild/pythoncore.vcxproj | 1 - PCbuild/pythoncore.vcxproj.filters | 3 - Python/pytime.c | 1 + Tools/build/stable_abi.py | 1 - Tools/c-analyzer/c_parser/preprocessor/gcc.py | 9 +- 25 files changed, 650 insertions(+), 667 deletions(-) delete mode 100644 Include/cpython/pytime.h create mode 100644 Misc/NEWS.d/next/C API/2023-07-01-21-23-33.gh-issue-106316.hp2Ijw.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 26efa5083672ec..c044e9b8f88c58 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -79,7 +79,7 @@ Doc/library/time.rst @pganssle @abalkin Lib/test/test_time.py @pganssle @abalkin Modules/timemodule.c @pganssle @abalkin Python/pytime.c @pganssle @abalkin -Include/pytime.h @pganssle @abalkin +Include/internal/pycore_time.h @pganssle @abalkin # Email and related **/*mail* @python/email-team diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9696dd4ff0b700..628945f7d5804f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -602,3 +602,6 @@ Removed use ``PyObject_Vectorcall()`` which is available since Python 3.8 (:pep:`590`). (Contributed by Victor Stinner in :gh:`106023`.) + +* Remove ``cpython/pytime.h`` header file: it only contained private functions. + (Contributed by Victor Stinner in :gh:`106316`.) diff --git a/Include/Python.h b/Include/Python.h index 45ba2ec12f2ad2..183d07cc539b4a 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -83,7 +83,6 @@ #include "weakrefobject.h" #include "structseq.h" #include "cpython/picklebufobject.h" -#include "cpython/pytime.h" #include "codecs.h" #include "pyerrors.h" #include "pythread.h" diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h deleted file mode 100644 index 16d88d191e9e25..00000000000000 --- a/Include/cpython/pytime.h +++ /dev/null @@ -1,331 +0,0 @@ -// The _PyTime_t API is written to use timestamp and timeout values stored in -// various formats and to read clocks. -// -// The _PyTime_t type is an integer to support directly common arithmetic -// operations like t1 + t2. -// -// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type -// is signed to support negative timestamps. The supported range is around -// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the -// supported date range is around [1677-09-21; 2262-04-11]. -// -// Formats: -// -// * seconds -// * seconds as a floating pointer number (C double) -// * milliseconds (10^-3 seconds) -// * microseconds (10^-6 seconds) -// * 100 nanoseconds (10^-7 seconds) -// * nanoseconds (10^-9 seconds) -// * timeval structure, 1 microsecond resolution (10^-6 seconds) -// * timespec structure, 1 nanosecond resolution (10^-9 seconds) -// -// Integer overflows are detected and raise OverflowError. Conversion to a -// resolution worse than 1 nanosecond is rounded correctly with the requested -// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling -// (towards +inf), half even and up (away from zero). -// -// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so -// the caller doesn't have to handle errors and doesn't need to hold the GIL. -// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on -// overflow. -// -// Clocks: -// -// * System clock -// * Monotonic clock -// * Performance counter -// -// Operations like (t * k / q) with integers are implemented in a way to reduce -// the risk of integer overflow. Such operation is used to convert a clock -// value expressed in ticks with a frequency to _PyTime_t, like -// QueryPerformanceCounter() with QueryPerformanceFrequency(). - -#ifndef Py_LIMITED_API -#ifndef Py_PYTIME_H -#define Py_PYTIME_H - -/************************************************************************** -Symbols and macros to supply platform-independent interfaces to time related -functions and constants -**************************************************************************/ -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __clang__ -struct timeval; -#endif - -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ -typedef int64_t _PyTime_t; -// _PyTime_MIN nanoseconds is around -292.3 years -#define _PyTime_MIN INT64_MIN -// _PyTime_MAX nanoseconds is around +292.3 years -#define _PyTime_MAX INT64_MAX -#define _SIZEOF_PYTIME_T 8 - -typedef enum { - /* Round towards minus infinity (-inf). - For example, used to read a clock. */ - _PyTime_ROUND_FLOOR=0, - /* Round towards infinity (+inf). - For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going to nearest even integer. - For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_EVEN=2, - /* Round away from zero - For example, used for timeout. _PyTime_ROUND_CEILING rounds - -1e-9 to 0 milliseconds which causes bpo-31786 issue. - _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps - the timeout sign as expected. select.poll(timeout) must block - for negative values." */ - _PyTime_ROUND_UP=3, - /* _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be - used for timeouts. */ - _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP -} _PyTime_round_t; - - -/* Convert a time_t to a PyLong. */ -PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( - time_t sec); - -/* Convert a PyLong to a time_t. */ -PyAPI_FUNC(time_t) _PyLong_AsTime_t( - PyObject *obj); - -/* Convert a number of seconds, int or float, to time_t. */ -PyAPI_FUNC(int) _PyTime_ObjectToTime_t( - PyObject *obj, - time_t *sec, - _PyTime_round_t); - -/* Convert a number of seconds, int or float, to a timeval structure. - usec is in the range [0; 999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000). */ -PyAPI_FUNC(int) _PyTime_ObjectToTimeval( - PyObject *obj, - time_t *sec, - long *usec, - _PyTime_round_t); - -/* Convert a number of seconds, int or float, to a timespec structure. - nsec is in the range [0; 999999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000000). */ -PyAPI_FUNC(int) _PyTime_ObjectToTimespec( - PyObject *obj, - time_t *sec, - long *nsec, - _PyTime_round_t); - - -/* Create a timestamp from a number of seconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); - -/* Macro to create a timestamp from a number of seconds, no integer overflow. - Only use the macro for small values, prefer _PyTime_FromSeconds(). */ -#define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -/* Create a timestamp from a number of nanoseconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); - -/* Create a timestamp from a number of microseconds. - * Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromMicrosecondsClamp(_PyTime_t us); - -/* Create a timestamp from nanoseconds (Python int). */ -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, - PyObject *obj); - -/* Convert a number of seconds (Python float or int) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds as a C double. */ -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - -/* Convert timestamp to a number of milliseconds (10^-3 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of microseconds (10^-6 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsNanoseconds(_PyTime_t t); - -#ifdef MS_WINDOWS -// Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -PyAPI_FUNC(_PyTime_t) _PyTime_As100Nanoseconds(_PyTime_t t, - _PyTime_round_t round); -#endif - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int - object. */ -PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); - -#ifndef MS_WINDOWS -/* Create a timestamp from a timeval structure. - Raise an exception and return -1 on overflow, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); -#endif - -/* Convert a timestamp to a timeval structure (microsecond resolution). - tv_usec is always positive. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds (secs) and microseconds (us). - us is always positive. This function is similar to _PyTime_AsTimeval() - except that secs is always a time_t type, whereas the timeval structure - uses a C long for tv_sec on Windows. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, - time_t *secs, - int *us, - _PyTime_round_t round); - -#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) -/* Create a timestamp from a timespec structure. - Raise an exception and return -1 on overflow, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); - -/* Convert a timestamp to a timespec structure (nanosecond resolution). - tv_nsec is always positive. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); - -/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); -#endif - - -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -PyAPI_FUNC(_PyTime_t) _PyTime_Add(_PyTime_t t1, _PyTime_t t2); - -/* Compute ticks * mul / div. - Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. - The caller must ensure that ((div - 1) * mul) cannot overflow. */ -PyAPI_FUNC(_PyTime_t) _PyTime_MulDiv(_PyTime_t ticks, - _PyTime_t mul, - _PyTime_t div); - -/* Structure used by time.get_clock_info() */ -typedef struct { - const char *implementation; - int monotonic; - int adjustable; - double resolution; -} _Py_clock_info_t; - -/* Get the current time from the system clock. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetSystemClockWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - -/* Get the current time from the system clock. - * On success, set *t and *info (if not NULL), and return 0. - * On error, raise an exception and return -1. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -/* Converts a timestamp to the Gregorian time, using the local time zone. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); - -/* Converts a timestamp to the Gregorian time, assuming UTC. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); - -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); - -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -// Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); - -// Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); - -#ifdef __cplusplus -} -#endif - -#endif /* Py_PYTIME_H */ -#endif /* Py_LIMITED_API */ diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 0a9f24efbdb908..ee93f7d99d9155 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -5,6 +5,8 @@ extern "C" { #endif +#include "pycore_time.h" // _PyTime_t + struct _import_runtime_state { /* The builtin modules (defined in config.c). */ diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 949170c4493799..3d394e8d36a132 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -1,3 +1,46 @@ +// The _PyTime_t API is written to use timestamp and timeout values stored in +// various formats and to read clocks. +// +// The _PyTime_t type is an integer to support directly common arithmetic +// operations like t1 + t2. +// +// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type +// is signed to support negative timestamps. The supported range is around +// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the +// supported date range is around [1677-09-21; 2262-04-11]. +// +// Formats: +// +// * seconds +// * seconds as a floating pointer number (C double) +// * milliseconds (10^-3 seconds) +// * microseconds (10^-6 seconds) +// * 100 nanoseconds (10^-7 seconds) +// * nanoseconds (10^-9 seconds) +// * timeval structure, 1 microsecond resolution (10^-6 seconds) +// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// +// Integer overflows are detected and raise OverflowError. Conversion to a +// resolution worse than 1 nanosecond is rounded correctly with the requested +// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling +// (towards +inf), half even and up (away from zero). +// +// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so +// the caller doesn't have to handle errors and doesn't need to hold the GIL. +// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on +// overflow. +// +// Clocks: +// +// * System clock +// * Monotonic clock +// * Performance counter +// +// Operations like (t * k / q) with integers are implemented in a way to reduce +// the risk of integer overflow. Such operation is used to convert a clock +// value expressed in ticks with a frequency to _PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency(). + #ifndef Py_INTERNAL_TIME_H #define Py_INTERNAL_TIME_H #ifdef __cplusplus @@ -19,6 +62,275 @@ struct _time_runtime_state { }; +#ifdef __clang__ +struct timeval; +#endif + +/* _PyTime_t: Python timestamp with subsecond precision. It can be used to + store a duration, and so indirectly a date (related to another date, like + UNIX epoch). */ +typedef int64_t _PyTime_t; +// _PyTime_MIN nanoseconds is around -292.3 years +#define _PyTime_MIN INT64_MIN +// _PyTime_MAX nanoseconds is around +292.3 years +#define _PyTime_MAX INT64_MAX +#define _SIZEOF_PYTIME_T 8 + +typedef enum { + /* Round towards minus infinity (-inf). + For example, used to read a clock. */ + _PyTime_ROUND_FLOOR=0, + /* Round towards infinity (+inf). + For example, used for timeout to wait "at least" N seconds. */ + _PyTime_ROUND_CEILING=1, + /* Round to nearest with ties going to nearest even integer. + For example, used to round from a Python float. */ + _PyTime_ROUND_HALF_EVEN=2, + /* Round away from zero + For example, used for timeout. _PyTime_ROUND_CEILING rounds + -1e-9 to 0 milliseconds which causes bpo-31786 issue. + _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps + the timeout sign as expected. select.poll(timeout) must block + for negative values." */ + _PyTime_ROUND_UP=3, + /* _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be + used for timeouts. */ + _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP +} _PyTime_round_t; + + +/* Convert a time_t to a PyLong. */ +PyAPI_FUNC(PyObject*) _PyLong_FromTime_t(time_t sec); + +/* Convert a PyLong to a time_t. */ +PyAPI_FUNC(time_t) _PyLong_AsTime_t(PyObject *obj); + +/* Convert a number of seconds, int or float, to time_t. */ +PyAPI_FUNC(int) _PyTime_ObjectToTime_t( + PyObject *obj, + time_t *sec, + _PyTime_round_t); + +/* Convert a number of seconds, int or float, to a timeval structure. + usec is in the range [0; 999999] and rounded towards zero. + For example, -1.2 is converted to (-2, 800000). */ +PyAPI_FUNC(int) _PyTime_ObjectToTimeval( + PyObject *obj, + time_t *sec, + long *usec, + _PyTime_round_t); + +/* Convert a number of seconds, int or float, to a timespec structure. + nsec is in the range [0; 999999999] and rounded towards zero. + For example, -1.2 is converted to (-2, 800000000). */ +PyAPI_FUNC(int) _PyTime_ObjectToTimespec( + PyObject *obj, + time_t *sec, + long *nsec, + _PyTime_round_t); + + +/* Create a timestamp from a number of seconds. */ +PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); + +/* Macro to create a timestamp from a number of seconds, no integer overflow. + Only use the macro for small values, prefer _PyTime_FromSeconds(). */ +#define _PYTIME_FROMSECONDS(seconds) \ + ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) + +/* Create a timestamp from a number of nanoseconds. */ +PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); + +/* Create a timestamp from a number of microseconds. + * Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. */ +extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); + +/* Create a timestamp from nanoseconds (Python int). */ +PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, + PyObject *obj); + +/* Convert a number of seconds (Python float or int) to a timestamp. + Raise an exception and return -1 on error, return 0 on success. */ +PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, + PyObject *obj, + _PyTime_round_t round); + +/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. + Raise an exception and return -1 on error, return 0 on success. */ +PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, + PyObject *obj, + _PyTime_round_t round); + +/* Convert a timestamp to a number of seconds as a C double. */ +PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); + +/* Convert timestamp to a number of milliseconds (10^-3 seconds). */ +PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, + _PyTime_round_t round); + +/* Convert timestamp to a number of microseconds (10^-6 seconds). */ +PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, + _PyTime_round_t round); + +/* Convert timestamp to a number of nanoseconds (10^-9 seconds). */ +PyAPI_FUNC(_PyTime_t) _PyTime_AsNanoseconds(_PyTime_t t); + +#ifdef MS_WINDOWS +// Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). +extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, + _PyTime_round_t round); +#endif + +/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int + object. */ +PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); + +#ifndef MS_WINDOWS +/* Create a timestamp from a timeval structure. + Raise an exception and return -1 on overflow, return 0 on success. */ +extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +#endif + +/* Convert a timestamp to a timeval structure (microsecond resolution). + tv_usec is always positive. + Raise an exception and return -1 if the conversion overflowed, + return 0 on success. */ +PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, + struct timeval *tv, + _PyTime_round_t round); + +/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. + On overflow, clamp tv_sec to _PyTime_t min/max. */ +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, + struct timeval *tv, + _PyTime_round_t round); + +/* Convert a timestamp to a number of seconds (secs) and microseconds (us). + us is always positive. This function is similar to _PyTime_AsTimeval() + except that secs is always a time_t type, whereas the timeval structure + uses a C long for tv_sec on Windows. + Raise an exception and return -1 if the conversion overflowed, + return 0 on success. */ +PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( + _PyTime_t t, + time_t *secs, + int *us, + _PyTime_round_t round); + +#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) +/* Create a timestamp from a timespec structure. + Raise an exception and return -1 on overflow, return 0 on success. */ +extern int _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); + +/* Convert a timestamp to a timespec structure (nanosecond resolution). + tv_nsec is always positive. + Raise an exception and return -1 on error, return 0 on success. */ +PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); + +/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. + On overflow, clamp tv_sec to _PyTime_t min/max. */ +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); +#endif + + +// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); + +/* Compute ticks * mul / div. + Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. + The caller must ensure that ((div - 1) * mul) cannot overflow. */ +extern _PyTime_t _PyTime_MulDiv(_PyTime_t ticks, + _PyTime_t mul, + _PyTime_t div); + +/* Structure used by time.get_clock_info() */ +typedef struct { + const char *implementation; + int monotonic; + int adjustable; + double resolution; +} _Py_clock_info_t; + +/* Get the current time from the system clock. + + If the internal clock fails, silently ignore the error and return 0. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. + + Use _PyTime_GetSystemClockWithInfo() to check for failure. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); + +/* Get the current time from the system clock. + * On success, set *t and *info (if not NULL), and return 0. + * On error, raise an exception and return -1. + */ +PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + +/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. + The clock is not affected by system clock updates. The reference point of + the returned value is undefined, so that only the difference between the + results of consecutive calls is valid. + + If the internal clock fails, silently ignore the error and return 0. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. + + Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); + +/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. + The clock is not affected by system clock updates. The reference point of + the returned value is undefined, so that only the difference between the + results of consecutive calls is valid. + + Fill info (if set) with information of the function used to get the time. + + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + + +/* Converts a timestamp to the Gregorian time, using the local time zone. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); + +/* Converts a timestamp to the Gregorian time, assuming UTC. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); + +/* Get the performance counter: clock with the highest available resolution to + measure a short duration. + + If the internal clock fails, silently ignore the error and return 0. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. + + Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); + +/* Get the performance counter: clock with the highest available resolution to + measure a short duration. + + Fill info (if set) with information of the function used to get the time. + + Return 0 on success, raise an exception and return -1 on error. */ +extern int _PyTime_GetPerfCounterWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + + +// Create a deadline. +// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); + +// Get remaining time from a deadline. +// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); + + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 02cc3f43a66a67..3b5640abdb6b89 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -14,6 +14,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test.support import skip_if_buggy_ucrt_strfptime @@ -761,7 +765,8 @@ def test_short_times(self): self.assertIs(lt.tm_zone, None) -@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +@unittest.skipIf(_testcapi is None, 'need the _testinternalcapi module') +@unittest.skipIf(_testinternalcapi is None, 'need the _testinternalcapi module') class CPyTimeTestCase: """ Base class to test the C _PyTime_t API. @@ -769,7 +774,7 @@ class CPyTimeTestCase: OVERFLOW_SECONDS = None def setUp(self): - from _testcapi import SIZEOF_TIME_T + from _testinternalcapi import SIZEOF_TIME_T bits = SIZEOF_TIME_T * 8 - 1 self.time_t_min = -2 ** bits self.time_t_max = 2 ** bits - 1 @@ -897,39 +902,39 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) def test_FromSeconds(self): - from _testcapi import PyTime_FromSeconds + from _testinternalcapi import _PyTime_FromSeconds - # PyTime_FromSeconds() expects a C int, reject values out of range + # _PyTime_FromSeconds() expects a C int, reject values out of range def c_int_filter(secs): return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) - self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + self.check_int_rounding(lambda secs, rnd: _PyTime_FromSeconds(secs), lambda secs: secs * SEC_TO_NS, value_filter=c_int_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(TypeError): - PyTime_FromSeconds(float('nan')) + _PyTime_FromSeconds(float('nan')) def test_FromSecondsObject(self): - from _testcapi import PyTime_FromSecondsObject + from _testinternalcapi import _PyTime_FromSecondsObject self.check_int_rounding( - PyTime_FromSecondsObject, + _PyTime_FromSecondsObject, lambda secs: secs * SEC_TO_NS) self.check_float_rounding( - PyTime_FromSecondsObject, + _PyTime_FromSecondsObject, lambda ns: self.decimal_round(ns * SEC_TO_NS)) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - PyTime_FromSecondsObject(float('nan'), time_rnd) + _PyTime_FromSecondsObject(float('nan'), time_rnd) def test_AsSecondsDouble(self): - from _testcapi import PyTime_AsSecondsDouble + from _testinternalcapi import _PyTime_AsSecondsDouble def float_converter(ns): if abs(ns) % SEC_TO_NS == 0: @@ -937,14 +942,14 @@ def float_converter(ns): else: return float(ns) / SEC_TO_NS - self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), + self.check_int_rounding(lambda ns, rnd: _PyTime_AsSecondsDouble(ns), float_converter, NS_TO_SEC) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(TypeError): - PyTime_AsSecondsDouble(float('nan')) + _PyTime_AsSecondsDouble(float('nan')) def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -956,7 +961,7 @@ def converter(value): return converter def test_AsTimeval(self): - from _testcapi import PyTime_AsTimeval + from _testinternalcapi import _PyTime_AsTimeval us_converter = self.create_decimal_converter(US_TO_NS) @@ -973,28 +978,28 @@ def seconds_filter(secs): else: seconds_filter = self.time_t_filter - self.check_int_rounding(PyTime_AsTimeval, + self.check_int_rounding(_PyTime_AsTimeval, timeval_converter, NS_TO_SEC, value_filter=seconds_filter) - @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), - 'need _testcapi.PyTime_AsTimespec') + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimespec'), + 'need _testinternalcapi._PyTime_AsTimespec') def test_AsTimespec(self): - from _testcapi import PyTime_AsTimespec + from _testinternalcapi import _PyTime_AsTimespec def timespec_converter(ns): return divmod(ns, SEC_TO_NS) - self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + self.check_int_rounding(lambda ns, rnd: _PyTime_AsTimespec(ns), timespec_converter, NS_TO_SEC, value_filter=self.time_t_filter) - @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimeval_clamp'), - 'need _testcapi.PyTime_AsTimeval_clamp') + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimeval_clamp'), + 'need _testinternalcapi._PyTime_AsTimeval_clamp') def test_AsTimeval_clamp(self): - from _testcapi import PyTime_AsTimeval_clamp + from _testinternalcapi import _PyTime_AsTimeval_clamp if sys.platform == 'win32': from _testcapi import LONG_MIN, LONG_MAX @@ -1005,7 +1010,7 @@ def test_AsTimeval_clamp(self): tv_sec_min = self.time_t_min for t in (_PyTime_MIN, _PyTime_MAX): - ts = PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) + ts = _PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) with decimal.localcontext() as context: context.rounding = decimal.ROUND_CEILING us = self.decimal_round(decimal.Decimal(t) / US_TO_NS) @@ -1018,13 +1023,13 @@ def test_AsTimeval_clamp(self): tv_usec = 0 self.assertEqual(ts, (tv_sec, tv_usec)) - @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec_clamp'), - 'need _testcapi.PyTime_AsTimespec_clamp') + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimespec_clamp'), + 'need _testinternalcapi._PyTime_AsTimespec_clamp') def test_AsTimespec_clamp(self): - from _testcapi import PyTime_AsTimespec_clamp + from _testinternalcapi import _PyTime_AsTimespec_clamp for t in (_PyTime_MIN, _PyTime_MAX): - ts = PyTime_AsTimespec_clamp(t) + ts = _PyTime_AsTimespec_clamp(t) tv_sec, tv_nsec = divmod(t, NS_TO_SEC) if self.time_t_max < tv_sec: tv_sec = self.time_t_max @@ -1035,16 +1040,16 @@ def test_AsTimespec_clamp(self): self.assertEqual(ts, (tv_sec, tv_nsec)) def test_AsMilliseconds(self): - from _testcapi import PyTime_AsMilliseconds + from _testinternalcapi import _PyTime_AsMilliseconds - self.check_int_rounding(PyTime_AsMilliseconds, + self.check_int_rounding(_PyTime_AsMilliseconds, self.create_decimal_converter(MS_TO_NS), NS_TO_SEC) def test_AsMicroseconds(self): - from _testcapi import PyTime_AsMicroseconds + from _testinternalcapi import _PyTime_AsMicroseconds - self.check_int_rounding(PyTime_AsMicroseconds, + self.check_int_rounding(_PyTime_AsMicroseconds, self.create_decimal_converter(US_TO_NS), NS_TO_SEC) @@ -1058,13 +1063,13 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): OVERFLOW_SECONDS = 2 ** 64 def test_object_to_time_t(self): - from _testcapi import pytime_object_to_time_t + from _testinternalcapi import _PyTime_ObjectToTime_t - self.check_int_rounding(pytime_object_to_time_t, + self.check_int_rounding(_PyTime_ObjectToTime_t, lambda secs: secs, value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_time_t, + self.check_float_rounding(_PyTime_ObjectToTime_t, self.decimal_round, value_filter=self.time_t_filter) @@ -1084,36 +1089,36 @@ def converter(secs): return converter def test_object_to_timeval(self): - from _testcapi import pytime_object_to_timeval + from _testinternalcapi import _PyTime_ObjectToTimeval - self.check_int_rounding(pytime_object_to_timeval, + self.check_int_rounding(_PyTime_ObjectToTimeval, lambda secs: (secs, 0), value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_timeval, + self.check_float_rounding(_PyTime_ObjectToTimeval, self.create_converter(SEC_TO_US), value_filter=self.time_t_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - pytime_object_to_timeval(float('nan'), time_rnd) + _PyTime_ObjectToTimeval(float('nan'), time_rnd) def test_object_to_timespec(self): - from _testcapi import pytime_object_to_timespec + from _testinternalcapi import _PyTime_ObjectToTimespec - self.check_int_rounding(pytime_object_to_timespec, + self.check_int_rounding(_PyTime_ObjectToTimespec, lambda secs: (secs, 0), value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_timespec, + self.check_float_rounding(_PyTime_ObjectToTimespec, self.create_converter(SEC_TO_NS), value_filter=self.time_t_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - pytime_object_to_timespec(float('nan'), time_rnd) + _PyTime_ObjectToTimespec(float('nan'), time_rnd) @unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") class TestTimeWeaklinking(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index c1b512b94765de..54d0516ad4423b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1710,7 +1710,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pystate.h \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ - $(srcdir)/Include/cpython/pytime.h \ $(srcdir)/Include/cpython/setobject.h \ $(srcdir)/Include/cpython/sysmodule.h \ $(srcdir)/Include/cpython/traceback.h \ diff --git a/Misc/NEWS.d/next/C API/2023-07-01-21-23-33.gh-issue-106316.hp2Ijw.rst b/Misc/NEWS.d/next/C API/2023-07-01-21-23-33.gh-issue-106316.hp2Ijw.rst new file mode 100644 index 00000000000000..733954eb8614f2 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-07-01-21-23-33.gh-issue-106316.hp2Ijw.rst @@ -0,0 +1,2 @@ +Remove ``cpython/pytime.h`` header file: it only contained private +functions. Patch by Victor Stinner. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index c228cd5642e75b..11a022e3d2044e 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -159,7 +159,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index db5be842b8a35c..69cc05135c2a72 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_time.h" // _PyTime_t #include "structmember.h" // PyMemberDef #include // offsetof() diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index d1991ac6b464f2..aaec0a61916948 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -28,7 +28,6 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); -int _PyTestCapi_Init_PyTime(PyObject *module); int _PyTestCapi_Init_DateTime(PyObject *module); int _PyTestCapi_Init_Docstring(PyObject *module); int _PyTestCapi_Init_Mem(PyObject *module); diff --git a/Modules/_testcapi/pytime.c b/Modules/_testcapi/pytime.c index 7422bafc30193a..e69de29bb2d1d6 100644 --- a/Modules/_testcapi/pytime.c +++ b/Modules/_testcapi/pytime.c @@ -1,274 +0,0 @@ -#include "parts.h" - -#ifdef MS_WINDOWS -# include // struct timeval -#endif - -static PyObject * -test_pytime_fromseconds(PyObject *self, PyObject *args) -{ - int seconds; - if (!PyArg_ParseTuple(args, "i", &seconds)) { - return NULL; - } - _PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsNanosecondsObject(ts); -} - -static int -check_time_rounding(int round) -{ - if (round != _PyTime_ROUND_FLOOR - && round != _PyTime_ROUND_CEILING - && round != _PyTime_ROUND_HALF_EVEN - && round != _PyTime_ROUND_UP) - { - PyErr_SetString(PyExc_ValueError, "invalid rounding"); - return -1; - } - return 0; -} - -static PyObject * -test_pytime_fromsecondsobject(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { - return NULL; - } - return _PyTime_AsNanosecondsObject(ts); -} - -static PyObject * -test_pytime_assecondsdouble(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t ts; - if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { - return NULL; - } - double d = _PyTime_AsSecondsDouble(ts); - return PyFloat_FromDouble(d); -} - -static PyObject * -test_PyTime_AsTimeval(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timeval tv; - if (_PyTime_AsTimeval(t, &tv, round) < 0) { - return NULL; - } - - PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); - if (seconds == NULL) { - return NULL; - } - return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); -} - -static PyObject * -test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timeval tv; - _PyTime_AsTimeval_clamp(t, &tv, round); - - PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); - if (seconds == NULL) { - return NULL; - } - return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); -} - -#ifdef HAVE_CLOCK_GETTIME -static PyObject * -test_PyTime_AsTimespec(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timespec ts; - if (_PyTime_AsTimespec(t, &ts) == -1) { - return NULL; - } - return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); -} - -static PyObject * -test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) -{ - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - struct timespec ts; - _PyTime_AsTimespec_clamp(t, &ts); - return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); -} -#endif - -static PyObject * -test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t ms = _PyTime_AsMilliseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(ms); - return _PyTime_AsNanosecondsObject(ns); -} - -static PyObject * -test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) -{ - PyObject *obj; - int round; - if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { - return NULL; - } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - _PyTime_t us = _PyTime_AsMicroseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(us); - return _PyTime_AsNanosecondsObject(ns); -} - -static PyObject * -test_pytime_object_to_time_t(PyObject *self, PyObject *args) -{ - PyObject *obj; - time_t sec; - int round; - if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_time_t", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - if (_PyTime_ObjectToTime_t(obj, &sec, round) == -1) { - return NULL; - } - return _PyLong_FromTime_t(sec); -} - -static PyObject * -test_pytime_object_to_timeval(PyObject *self, PyObject *args) -{ - PyObject *obj; - time_t sec; - long usec; - int round; - if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_timeval", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - if (_PyTime_ObjectToTimeval(obj, &sec, &usec, round) == -1) { - return NULL; - } - return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), usec); -} - -static PyObject * -test_pytime_object_to_timespec(PyObject *self, PyObject *args) -{ - PyObject *obj; - time_t sec; - long nsec; - int round; - if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_timespec", &obj, &round)) { - return NULL; - } - if (check_time_rounding(round) < 0) { - return NULL; - } - if (_PyTime_ObjectToTimespec(obj, &sec, &nsec, round) == -1) { - return NULL; - } - return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); -} - -static PyMethodDef test_methods[] = { - {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, - {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, - {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, -#ifdef HAVE_CLOCK_GETTIME - {"PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, - {"PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, -#endif - {"PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS}, - {"PyTime_AsTimeval_clamp", test_PyTime_AsTimeval_clamp, METH_VARARGS}, - {"PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS}, - {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, - {"pytime_object_to_time_t", test_pytime_object_to_time_t, METH_VARARGS}, - {"pytime_object_to_timespec", test_pytime_object_to_timespec, METH_VARARGS}, - {"pytime_object_to_timeval", test_pytime_object_to_timeval, METH_VARARGS}, - {NULL}, -}; - -int -_PyTestCapi_Init_PyTime(PyObject *mod) -{ - if (PyModule_AddFunctions(mod, test_methods) < 0) { - return -1; - } - return 0; -} diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index dc8acec6c76921..ec2e64f2f46fc1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4216,9 +4216,6 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_GetArgs(m) < 0) { return NULL; } - if (_PyTestCapi_Init_PyTime(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_DateTime(m) < 0) { return NULL; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3c0c2adaf6f843..c598d7edef8ae2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -31,6 +31,10 @@ #include "clinic/_testinternalcapi.c.h" +#ifdef MS_WINDOWS +# include // struct timeval +#endif + #define MODULE_NAME "_testinternalcapi" @@ -969,6 +973,249 @@ pending_identify(PyObject *self, PyObject *args) } +static PyObject * +test_pytime_fromseconds(PyObject *self, PyObject *args) +{ + int seconds; + if (!PyArg_ParseTuple(args, "i", &seconds)) { + return NULL; + } + _PyTime_t ts = _PyTime_FromSeconds(seconds); + return _PyTime_AsNanosecondsObject(ts); +} + +static int +check_time_rounding(int round) +{ + if (round != _PyTime_ROUND_FLOOR + && round != _PyTime_ROUND_CEILING + && round != _PyTime_ROUND_HALF_EVEN + && round != _PyTime_ROUND_UP) + { + PyErr_SetString(PyExc_ValueError, "invalid rounding"); + return -1; + } + return 0; +} + +static PyObject * +test_pytime_fromsecondsobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t ts; + if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { + return NULL; + } + return _PyTime_AsNanosecondsObject(ts); +} + +static PyObject * +test_pytime_assecondsdouble(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t ts; + if (_PyTime_FromNanosecondsObject(&ts, obj) < 0) { + return NULL; + } + double d = _PyTime_AsSecondsDouble(ts); + return PyFloat_FromDouble(d); +} + +static PyObject * +test_PyTime_AsTimeval(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timeval tv; + if (_PyTime_AsTimeval(t, &tv, round) < 0) { + return NULL; + } + + PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); + if (seconds == NULL) { + return NULL; + } + return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); +} + +static PyObject * +test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timeval tv; + _PyTime_AsTimeval_clamp(t, &tv, round); + + PyObject *seconds = PyLong_FromLongLong(tv.tv_sec); + if (seconds == NULL) { + return NULL; + } + return Py_BuildValue("Nl", seconds, (long)tv.tv_usec); +} + +#ifdef HAVE_CLOCK_GETTIME +static PyObject * +test_PyTime_AsTimespec(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timespec ts; + if (_PyTime_AsTimespec(t, &ts) == -1) { + return NULL; + } + return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); +} + +static PyObject * +test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + struct timespec ts; + _PyTime_AsTimespec_clamp(t, &ts); + return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec); +} +#endif + +static PyObject * +test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t ms = _PyTime_AsMilliseconds(t, round); + _PyTime_t ns = _PyTime_FromNanoseconds(ms); + return _PyTime_AsNanosecondsObject(ns); +} + +static PyObject * +test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) +{ + PyObject *obj; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + _PyTime_t t; + if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + _PyTime_t us = _PyTime_AsMicroseconds(t, round); + _PyTime_t ns = _PyTime_FromNanoseconds(us); + return _PyTime_AsNanosecondsObject(ns); +} + +static PyObject * +test_pytime_object_to_time_t(PyObject *self, PyObject *args) +{ + PyObject *obj; + time_t sec; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + if (_PyTime_ObjectToTime_t(obj, &sec, round) == -1) { + return NULL; + } + return _PyLong_FromTime_t(sec); +} + +static PyObject * +test_pytime_object_to_timeval(PyObject *self, PyObject *args) +{ + PyObject *obj; + time_t sec; + long usec; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + if (_PyTime_ObjectToTimeval(obj, &sec, &usec, round) == -1) { + return NULL; + } + return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), usec); +} + +static PyObject * +test_pytime_object_to_timespec(PyObject *self, PyObject *args) +{ + PyObject *obj; + time_t sec; + long nsec; + int round; + if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { + return NULL; + } + if (check_time_rounding(round) < 0) { + return NULL; + } + if (_PyTime_ObjectToTimespec(obj, &sec, &nsec, round) == -1) { + return NULL; + } + return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1005,6 +1252,20 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS}, // {"pending_fd_identify", pending_fd_identify, METH_VARARGS, NULL}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, + {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, + {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, + {"_PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, +#ifdef HAVE_CLOCK_GETTIME + {"_PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS}, + {"_PyTime_AsTimespec_clamp", test_PyTime_AsTimespec_clamp, METH_VARARGS}, +#endif + {"_PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS}, + {"_PyTime_AsTimeval_clamp", test_PyTime_AsTimeval_clamp, METH_VARARGS}, + {"_PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS}, + {"_PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, + {"_PyTime_ObjectToTime_t", test_pytime_object_to_time_t, METH_VARARGS}, + {"_PyTime_ObjectToTimespec", test_pytime_object_to_timespec, METH_VARARGS}, + {"_PyTime_ObjectToTimeval", test_pytime_object_to_timeval, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -1019,6 +1280,11 @@ module_exec(PyObject *module) return 1; } + if (PyModule_AddObject(module, "SIZEOF_TIME_T", + PyLong_FromSsize_t(sizeof(time_t))) < 0) { + return 1; + } + return 0; } diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index dca7abff89146e..922b41005e8419 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -8,6 +8,7 @@ //#include #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() +#include "pycore_time.h" // _PyTime_t typedef struct { diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 9a4943c9eb2f75..7ab0804ad27233 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -14,6 +14,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() +#include "pycore_time.h" // _PyTime_t #include "structmember.h" // PyMemberDef #ifdef HAVE_SYS_DEVPOLL_H diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index f5ca00450ee92a..663ae3d6e0dd6c 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -1,5 +1,7 @@ /* Socket module header file */ +#include "pycore_time.h" // _PyTime_t + /* Includes needed for the sockaddr_* symbols below */ #ifndef MS_WINDOWS #ifdef __VMS diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 3db9426d1a25ff..de17d74c52e56f 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -100,7 +100,6 @@ - @@ -131,4 +130,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 8df4874659fa1c..637f7178d39d0e 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -30,9 +30,6 @@ Source Files - - Source Files - Source Files @@ -75,4 +72,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c99bc905c89e21..58abd871376a06 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -176,7 +176,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 53e89457f72a37..4d7b5ccf9a8a9e 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -447,9 +447,6 @@ Include\cpython - - Include\cpython - Include\cpython diff --git a/Python/pytime.c b/Python/pytime.c index d36d4417dabb22..49cd5f4e8ea617 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # include // struct timeval #endif diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 4cd1cd953d0d29..7cba788ff33578 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -40,7 +40,6 @@ "longintrepr.h", "parsetok.h", "pyatomic.h", - "pytime.h", "token.h", "ucnhash.h", } diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 147615707a3cb8..0929f7111eac92 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -60,6 +60,13 @@ def preprocess(filename, if not cwd or not os.path.isabs(cwd): cwd = os.path.abspath(cwd or '.') filename = _normpath(filename, cwd) + + postargs = POST_ARGS + if os.path.basename(filename) == 'socketmodule.h': + # Modules/socketmodule.h uses pycore_time.h which needs Py_BUILD_CORE. + # Usually it's defined by the C file which includes it. + postargs += ('-DPy_BUILD_CORE=1',) + text = _common.preprocess( TOOL, filename, @@ -67,7 +74,7 @@ def preprocess(filename, includes=includes, macros=macros, #preargs=PRE_ARGS, - postargs=POST_ARGS, + postargs=postargs, executable=['gcc'], compiler='unix', cwd=cwd, From 0530f4f64629ff97f3feb7524da0833b9535e8b6 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sun, 2 Jul 2023 01:46:06 +0300 Subject: [PATCH 03/31] gh-102541: Fix Helper.help("mod") for non-existent mod (#105934) If the output arg to Helper() is a stream rather than the default None, which means 'page to stdout', the ImportError from pydoc.resolve is currently not caught in pydoc.doc. The same error is caught when output is None. --------- Co-authored-by: Terry Jan Reedy --- Lib/pydoc.py | 6 +++++- Lib/test/test_pydoc.py | 7 +++++++ .../Library/2023-07-01-16-40-54.gh-issue-102541.C1ahtk.rst | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-01-16-40-54.gh-issue-102541.C1ahtk.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d69369d4196eaa..185f09e603df2e 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1790,7 +1790,11 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0, raise print(exc) else: - output.write(render_doc(thing, title, forceload, plaintext)) + try: + s = render_doc(thing, title, forceload, plaintext) + except ImportError as exc: + s = str(exc) + output.write(s) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index cefc71cb5a7f54..115ffd3c29d4d6 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -631,6 +631,13 @@ def test_builtin_on_metaclasses(self): # Testing that the subclasses section does not appear self.assertNotIn('Built-in subclasses', text) + def test_fail_help_output_redirect(self): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help("abd") + expected = missing_pattern % "abd" + self.assertEqual(expected, buf.getvalue().strip().replace('\n', os.linesep)) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @requires_docstrings diff --git a/Misc/NEWS.d/next/Library/2023-07-01-16-40-54.gh-issue-102541.C1ahtk.rst b/Misc/NEWS.d/next/Library/2023-07-01-16-40-54.gh-issue-102541.C1ahtk.rst new file mode 100644 index 00000000000000..efaf5db10f3e1c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-01-16-40-54.gh-issue-102541.C1ahtk.rst @@ -0,0 +1 @@ +Make pydoc.doc catch bad module ImportError when output stream is not None. From 18b1fdebe0cd5e601aa341227c13ec9d89bdf32c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 01:44:07 +0200 Subject: [PATCH 04/31] gh-106320: Remove _PyInterpreterState_Get() alias (#106321) Replace calls to the (removed) slow _PyInterpreterState_Get() with fast inlined _PyInterpreterState_GET() function. --- Doc/whatsnew/3.13.rst | 7 +++++++ Include/cpython/pystate.h | 5 +---- Include/internal/pycore_pystate.h | 2 +- ...3-07-02-00-00-20.gh-issue-106320.tZWcvG.rst | 3 +++ Modules/_threadmodule.c | 2 +- Objects/typeobject.c | 4 ++-- Objects/unicodeobject.c | 2 +- Python/ceval_gil.c | 4 ++-- Python/hamt.c | 2 +- Python/import.c | 2 +- Python/instrumentation.c | 18 +++++++++--------- 11 files changed, 29 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-07-02-00-00-20.gh-issue-106320.tZWcvG.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 628945f7d5804f..8f737c5935f67b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -605,3 +605,10 @@ Removed * Remove ``cpython/pytime.h`` header file: it only contained private functions. (Contributed by Victor Stinner in :gh:`106316`.) + +* Remove ``_PyInterpreterState_Get()`` alias to + :c:func:`PyInterpreterState_Get()` which was kept for backward compatibility + with Python 3.8. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. + (Contributed by Victor Stinner in :gh:`106320`.) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index f33c72d4cf4d2a..7d9e41bc2dd7ab 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -261,9 +261,6 @@ struct _ts { /* other API */ -// Alias for backward compatibility with Python 3.8 -#define _PyInterpreterState_Get PyInterpreterState_Get - /* An alias for the internal _PyThreadState_New(), kept for stable ABI compatibility. */ PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *); @@ -295,7 +292,7 @@ PyAPI_FUNC(int) PyGILState_Check(void); This function doesn't check for error. Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini() is called. - See also _PyInterpreterState_Get() and _PyInterpreterState_GET(). */ + See also PyInterpreterState_Get() and _PyInterpreterState_GET(). */ PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); /* The implementation of sys._current_frames() Returns a dict mapping diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 43652c4405ec1a..63fc6b2129ce2a 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -108,7 +108,7 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) The caller must hold the GIL. - See also _PyInterpreterState_Get() + See also PyInterpreterState_Get() and _PyGILState_GetInterpreterStateUnsafe(). */ static inline PyInterpreterState* _PyInterpreterState_GET(void) { PyThreadState *tstate = _PyThreadState_GET(); diff --git a/Misc/NEWS.d/next/C API/2023-07-02-00-00-20.gh-issue-106320.tZWcvG.rst b/Misc/NEWS.d/next/C API/2023-07-02-00-00-20.gh-issue-106320.tZWcvG.rst new file mode 100644 index 00000000000000..145d2ce7b0ceb1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-07-02-00-00-20.gh-issue-106320.tZWcvG.rst @@ -0,0 +1,3 @@ +Remove ``_PyInterpreterState_Get()`` alias to +:c:func:`PyInterpreterState_Get()` which was kept for backward compatibility +with Python 3.8. Patch by Victor Stinner. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 7f8b3cbbfcec7b..f570b4e7007156 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1104,7 +1104,7 @@ thread_run(void *boot_raw) static PyObject * thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->feature_flags & Py_RTFLAGS_DAEMON_THREADS) { Py_RETURN_TRUE; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e67945db9af330..3d3a63a75bd2fb 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6362,7 +6362,7 @@ object___reduce_ex___impl(PyObject *self, int protocol) /*[clinic end generated code: output=2e157766f6b50094 input=f326b43fb8a4c5ff]*/ { #define objreduce \ - (_Py_INTERP_CACHED_OBJECT(_PyInterpreterState_Get(), objreduce)) + (_Py_INTERP_CACHED_OBJECT(_PyInterpreterState_GET(), objreduce)) PyObject *reduce, *res; if (objreduce == NULL) { @@ -9688,7 +9688,7 @@ resolve_slotdups(PyTypeObject *type, PyObject *name) /* XXX Maybe this could be optimized more -- but is it worth it? */ /* pname and ptrs act as a little cache */ - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); #define pname _Py_INTERP_CACHED_OBJECT(interp, type_slots_pname) #define ptrs _Py_INTERP_CACHED_OBJECT(interp, type_slots_ptrs) pytype_slotdef *p, **pp; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4434bf19d43b70..74def5ada5134e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5831,7 +5831,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, PyObject *errorHandler = NULL; PyObject *exc = NULL; _PyUnicode_Name_CAPI *ucnhash_capi; - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index bb1279f46cf9f7..c6b1f9ef689ee1 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -491,7 +491,7 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); gil->interval = microseconds; @@ -499,7 +499,7 @@ void _PyEval_SetSwitchInterval(unsigned long microseconds) unsigned long _PyEval_GetSwitchInterval(void) { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); return gil->interval; diff --git a/Python/hamt.c b/Python/hamt.c index 8cb94641bef251..c78b5a7fab94f0 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -2425,7 +2425,7 @@ hamt_alloc(void) } #define _empty_hamt \ - (&_Py_INTERP_SINGLETON(_PyInterpreterState_Get(), hamt_empty)) + (&_Py_INTERP_SINGLETON(_PyInterpreterState_GET(), hamt_empty)) PyHamtObject * _PyHamt_New(void) diff --git a/Python/import.c b/Python/import.c index 324fe3812bdd49..fcc0346e7f15af 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1123,7 +1123,7 @@ check_multi_interp_extensions(PyInterpreterState *interp) int _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (check_multi_interp_extensions(interp)) { assert(!_Py_IsMainInterpreter(interp)); PyErr_Format(PyExc_ImportError, diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 86d014ededc8ee..3253a0ea13033c 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1219,7 +1219,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyObject * _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) { - PyInterpreterState *is = _PyInterpreterState_Get(); + PyInterpreterState *is = _PyInterpreterState_GET(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); PyObject *callback = is->monitoring_callables[tool_id][event_id]; @@ -1678,7 +1678,7 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; @@ -1696,7 +1696,7 @@ int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; @@ -1758,7 +1758,7 @@ monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) PyErr_SetString(PyExc_ValueError, "tool name must be a str"); return NULL; } - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->monitoring_tool_names[tool_id] != NULL) { PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id); return NULL; @@ -1782,7 +1782,7 @@ monitoring_free_tool_id_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return NULL; } - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); Py_CLEAR(interp->monitoring_tool_names[tool_id]); Py_RETURN_NONE; } @@ -1804,7 +1804,7 @@ monitoring_get_tool_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return NULL; } - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *name = interp->monitoring_tool_names[tool_id]; if (name == NULL) { Py_RETURN_NONE; @@ -1865,7 +1865,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return -1; } - _Py_Monitors *m = &_PyInterpreterState_Get()->monitors; + _Py_Monitors *m = &_PyInterpreterState_GET()->monitors; _PyMonitoringEventSet event_set = get_events(m, tool_id); return event_set; } @@ -1990,7 +1990,7 @@ monitoring_restart_events_impl(PyObject *module) * last restart version > instrumented version for all code objects * last restart version < current version */ - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); interp->last_restart_version = interp->monitoring_version + 1; interp->monitoring_version = interp->last_restart_version + 1; if (instrument_all_executing_code_objects(interp)) { @@ -2038,7 +2038,7 @@ static PyObject * monitoring__all_events_impl(PyObject *module) /*[clinic end generated code: output=6b7581e2dbb690f6 input=62ee9672c17b7f0e]*/ { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *res = PyDict_New(); if (res == NULL) { return NULL; From feb51f3a6443d7c0148e2e7be2ed58b4c69fa265 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 02:49:18 +0200 Subject: [PATCH 05/31] gh-106320: Remove private _PyTraceMalloc C API functions (#106324) * Remove private _PyTraceMalloc C API functions: move them to the internal C API. * Don't export most of these functions anymore, but still export _PyTraceMalloc_GetTraceback() used by tests. * Rename Include/tracemalloc.h to Include/cpython/tracemalloc.h --- Include/Python.h | 2 +- Include/cpython/tracemalloc.h | 26 ++++++++++ Include/internal/pycore_tracemalloc.h | 45 +++++++++++++++++ Include/tracemalloc.h | 72 --------------------------- Lib/test/test_tracemalloc.py | 4 +- Makefile.pre.in | 2 +- Modules/_testcapi/mem.c | 18 ------- Modules/_testinternalcapi.c | 20 +++++++- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 6 +-- 10 files changed, 99 insertions(+), 98 deletions(-) create mode 100644 Include/cpython/tracemalloc.h delete mode 100644 Include/tracemalloc.h diff --git a/Include/Python.h b/Include/Python.h index 183d07cc539b4a..07f6c202a7f126 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -103,7 +103,7 @@ #include "pystrcmp.h" #include "fileutils.h" #include "cpython/pyfpe.h" -#include "tracemalloc.h" +#include "cpython/tracemalloc.h" #include "cpython/optimizer.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/cpython/tracemalloc.h b/Include/cpython/tracemalloc.h new file mode 100644 index 00000000000000..61a16ea9a9f3eb --- /dev/null +++ b/Include/cpython/tracemalloc.h @@ -0,0 +1,26 @@ +#ifndef Py_LIMITED_API +#ifndef Py_TRACEMALLOC_H +#define Py_TRACEMALLOC_H + +/* Track an allocated memory block in the tracemalloc module. + Return 0 on success, return -1 on error (failed to allocate memory to store + the trace). + + Return -2 if tracemalloc is disabled. + + If memory block is already tracked, update the existing trace. */ +PyAPI_FUNC(int) PyTraceMalloc_Track( + unsigned int domain, + uintptr_t ptr, + size_t size); + +/* Untrack an allocated memory block in the tracemalloc module. + Do nothing if the block was not tracked. + + Return -2 if tracemalloc is disabled, otherwise return 0. */ +PyAPI_FUNC(int) PyTraceMalloc_Untrack( + unsigned int domain, + uintptr_t ptr); + +#endif // !Py_TRACEMALLOC_H +#endif // !Py_LIMITED_API diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index d086adc61c319b..cfc4d1fe43999e 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -117,6 +117,51 @@ struct _tracemalloc_runtime_state { } +/* Get the traceback where a memory block was allocated. + + Return a tuple of (filename: str, lineno: int) tuples. + + Return None if the tracemalloc module is disabled or if the memory block + is not tracked by tracemalloc. + + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( + unsigned int domain, + uintptr_t ptr); + +/* Return non-zero if tracemalloc is tracing */ +extern int _PyTraceMalloc_IsTracing(void); + +/* Clear the tracemalloc traces */ +extern void _PyTraceMalloc_ClearTraces(void); + +/* Clear the tracemalloc traces */ +extern PyObject* _PyTraceMalloc_GetTraces(void); + +/* Clear tracemalloc traceback for an object */ +extern PyObject* _PyTraceMalloc_GetObjectTraceback(PyObject *obj); + +/* Initialize tracemalloc */ +extern int _PyTraceMalloc_Init(void); + +/* Start tracemalloc */ +extern int _PyTraceMalloc_Start(int max_nframe); + +/* Stop tracemalloc */ +extern void _PyTraceMalloc_Stop(void); + +/* Get the tracemalloc traceback limit */ +extern int _PyTraceMalloc_GetTracebackLimit(void); + +/* Get the memory usage of tracemalloc in bytes */ +extern size_t _PyTraceMalloc_GetMemory(void); + +/* Get the current size and peak size of traced memory blocks as a 2-tuple */ +extern PyObject* _PyTraceMalloc_GetTracedMemory(void); + +/* Set the peak size of traced memory blocks to the current size */ +extern void _PyTraceMalloc_ResetPeak(void); + #ifdef __cplusplus } #endif diff --git a/Include/tracemalloc.h b/Include/tracemalloc.h deleted file mode 100644 index 580027a8e365e5..00000000000000 --- a/Include/tracemalloc.h +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef Py_TRACEMALLOC_H -#define Py_TRACEMALLOC_H - -#ifndef Py_LIMITED_API -/* Track an allocated memory block in the tracemalloc module. - Return 0 on success, return -1 on error (failed to allocate memory to store - the trace). - - Return -2 if tracemalloc is disabled. - - If memory block is already tracked, update the existing trace. */ -PyAPI_FUNC(int) PyTraceMalloc_Track( - unsigned int domain, - uintptr_t ptr, - size_t size); - -/* Untrack an allocated memory block in the tracemalloc module. - Do nothing if the block was not tracked. - - Return -2 if tracemalloc is disabled, otherwise return 0. */ -PyAPI_FUNC(int) PyTraceMalloc_Untrack( - unsigned int domain, - uintptr_t ptr); - -/* Get the traceback where a memory block was allocated. - - Return a tuple of (filename: str, lineno: int) tuples. - - Return None if the tracemalloc module is disabled or if the memory block - is not tracked by tracemalloc. - - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( - unsigned int domain, - uintptr_t ptr); - -/* Return non-zero if tracemalloc is tracing */ -PyAPI_FUNC(int) _PyTraceMalloc_IsTracing(void); - -/* Clear the tracemalloc traces */ -PyAPI_FUNC(void) _PyTraceMalloc_ClearTraces(void); - -/* Clear the tracemalloc traces */ -PyAPI_FUNC(PyObject *) _PyTraceMalloc_GetTraces(void); - -/* Clear tracemalloc traceback for an object */ -PyAPI_FUNC(PyObject *) _PyTraceMalloc_GetObjectTraceback(PyObject *obj); - -/* Initialize tracemalloc */ -PyAPI_FUNC(int) _PyTraceMalloc_Init(void); - -/* Start tracemalloc */ -PyAPI_FUNC(int) _PyTraceMalloc_Start(int max_nframe); - -/* Stop tracemalloc */ -PyAPI_FUNC(void) _PyTraceMalloc_Stop(void); - -/* Get the tracemalloc traceback limit */ -PyAPI_FUNC(int) _PyTraceMalloc_GetTracebackLimit(void); - -/* Get the memory usage of tracemalloc in bytes */ -PyAPI_FUNC(size_t) _PyTraceMalloc_GetMemory(void); - -/* Get the current size and peak size of traced memory blocks as a 2-tuple */ -PyAPI_FUNC(PyObject *) _PyTraceMalloc_GetTracedMemory(void); - -/* Set the peak size of traced memory blocks to the current size */ -PyAPI_FUNC(void) _PyTraceMalloc_ResetPeak(void); - -#endif - -#endif /* !Py_TRACEMALLOC_H */ diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 94bcee302fe730..4af4ca3b977236 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -11,8 +11,10 @@ try: import _testcapi + import _testinternalcapi except ImportError: _testcapi = None + _testinternalcapi = None EMPTY_STRING_SIZE = sys.getsizeof(b'') @@ -1008,7 +1010,7 @@ def tearDown(self): tracemalloc.stop() def get_traceback(self): - frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) + frames = _testinternalcapi._PyTraceMalloc_GetTraceback(self.domain, self.ptr) if frames is not None: return tracemalloc.Traceback(frames) else: diff --git a/Makefile.pre.in b/Makefile.pre.in index 54d0516ad4423b..e788e590dcbb43 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1658,7 +1658,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/structseq.h \ $(srcdir)/Include/sysmodule.h \ $(srcdir)/Include/traceback.h \ - $(srcdir)/Include/tracemalloc.h \ $(srcdir)/Include/tupleobject.h \ $(srcdir)/Include/unicodeobject.h \ $(srcdir)/Include/warnings.h \ @@ -1713,6 +1712,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/setobject.h \ $(srcdir)/Include/cpython/sysmodule.h \ $(srcdir)/Include/cpython/traceback.h \ + $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ $(srcdir)/Include/cpython/unicodeobject.h \ $(srcdir)/Include/cpython/warnings.h \ diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index af32e9668dda2d..16bda66af554af 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -655,23 +655,6 @@ tracemalloc_untrack(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -tracemalloc_get_traceback(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { - return NULL; - } - void *ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) { - return NULL; - } - - return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); -} - static PyMethodDef test_methods[] = { {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, @@ -697,7 +680,6 @@ static PyMethodDef test_methods[] = { // Tracemalloc tests {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, - {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c598d7edef8ae2..f6ae389ea05679 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1216,6 +1216,24 @@ test_pytime_object_to_timespec(PyObject *self, PyObject *args) } +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1250,7 +1268,6 @@ static PyMethodDef module_functions[] = { {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, -// {"pending_fd_identify", pending_fd_identify, METH_VARARGS, NULL}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, {"_PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"_PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, @@ -1266,6 +1283,7 @@ static PyMethodDef module_functions[] = { {"_PyTime_ObjectToTime_t", test_pytime_object_to_time_t, METH_VARARGS}, {"_PyTime_ObjectToTimespec", test_pytime_object_to_timespec, METH_VARARGS}, {"_PyTime_ObjectToTimeval", test_pytime_object_to_timeval, METH_VARARGS}, + {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 58abd871376a06..436381c3ea5db4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -179,6 +179,7 @@ + @@ -320,7 +321,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4d7b5ccf9a8a9e..4d9acf4893872d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -219,9 +219,6 @@ Include - - Include - Include @@ -453,6 +450,9 @@ Include\cpython + + Include + Include\cpython From 8571b271e7d16fe87d669a2e1e50f5ae3732bb31 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 03:39:38 +0200 Subject: [PATCH 06/31] gh-106320: Remove private _PyInterpreterState functions (#106325) Remove private _PyThreadState and _PyInterpreterState C API functions: move them to the internal C API (pycore_pystate.h and pycore_interp.h). Don't export most of these functions anymore, but still export functions used by tests. Remove _PyThreadState_Prealloc() and _PyThreadState_Init() from the C API, but keep it in the stable API. --- Include/cpython/pystate.h | 57 ------------------------------ Include/internal/pycore_interp.h | 37 +++++++++++++++++++ Include/internal/pycore_pystate.h | 19 ++++++++-- Modules/_testcapimodule.c | 5 --- Modules/_testinternalcapi.c | 22 ++++++++++++ Modules/_xxsubinterpretersmodule.c | 6 +++- Python/compile.c | 1 + Python/frozenmain.c | 3 +- Python/pystate.c | 6 ++-- 9 files changed, 86 insertions(+), 70 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7d9e41bc2dd7ab..e08bcdaf197628 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -40,8 +40,6 @@ PyAPI_FUNC(int) _PyInterpreterState_HasFeature(PyInterpreterState *interp, PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int); -PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *); - /* State unique per thread */ @@ -261,16 +259,10 @@ struct _ts { /* other API */ -/* An alias for the internal _PyThreadState_New(), - kept for stable ABI compatibility. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *); - /* Similar to PyThreadState_Get(), but don't issue a fatal error * if it is NULL. */ PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); -PyAPI_FUNC(PyObject *) _PyThreadState_GetDict(PyThreadState *tstate); - // Disable tracing and profiling. PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); @@ -295,16 +287,6 @@ PyAPI_FUNC(int) PyGILState_Check(void); See also PyInterpreterState_Get() and _PyInterpreterState_GET(). */ PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); -/* The implementation of sys._current_frames() Returns a dict mapping - thread id to that thread's current frame. -*/ -PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void); - -/* The implementation of sys._current_exceptions() Returns a dict mapping - thread id to that thread's current exception. -*/ -PyAPI_FUNC(PyObject *) _PyThread_CurrentExceptions(void); - /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); @@ -324,45 +306,6 @@ PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); -PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp); - -/* Get a copy of the current interpreter configuration. - - Return 0 on success. Raise an exception and return -1 on error. - - The caller must initialize 'config', using PyConfig_InitPythonConfig() - for example. - - Python must be preinitialized to call this method. - The caller must hold the GIL. - - Once done with the configuration, PyConfig_Clear() must be called to clear - it. */ -PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( - struct PyConfig *config); - -/* Set the configuration of the current interpreter. - - This function should be called during or just after the Python - initialization. - - Update the sys module with the new configuration. If the sys module was - modified directly after the Python initialization, these changes are lost. - - Some configuration like faulthandler or warnoptions can be updated in the - configuration, but don't reconfigure Python (don't enable/disable - faulthandler and don't reconfigure warnings filters). - - Return 0 on success. Raise an exception and return -1 on error. - - The configuration should come from _PyInterpreterState_GetConfigCopy(). */ -PyAPI_FUNC(int) _PyInterpreterState_SetConfig( - const struct PyConfig *config); - -// Get the configuration of the current interpreter. -// The caller must hold the GIL. -PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); - /* cross-interpreter data */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 466ae6fbbdc4eb..a1f00df12d6ac9 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -238,6 +238,43 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(PyObject*) _PyInterpreterState_GetMainModule(PyInterpreterState *); + +extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); + +/* Get a copy of the current interpreter configuration. + + Return 0 on success. Raise an exception and return -1 on error. + + The caller must initialize 'config', using PyConfig_InitPythonConfig() + for example. + + Python must be preinitialized to call this method. + The caller must hold the GIL. + + Once done with the configuration, PyConfig_Clear() must be called to clear + it. */ +PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( + struct PyConfig *config); + +/* Set the configuration of the current interpreter. + + This function should be called during or just after the Python + initialization. + + Update the sys module with the new configuration. If the sys module was + modified directly after the Python initialization, these changes are lost. + + Some configuration like faulthandler or warnoptions can be updated in the + configuration, but don't reconfigure Python (don't enable/disable + faulthandler and don't reconfigure warnings filters). + + Return 0 on success. Raise an exception and return -1 on error. + + The configuration should come from _PyInterpreterState_GetConfigCopy(). */ +PyAPI_FUNC(int) _PyInterpreterState_SetConfig( + const struct PyConfig *config); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 63fc6b2129ce2a..0659084194d293 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -123,9 +123,6 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { PyAPI_FUNC(PyThreadState *) _PyThreadState_New(PyInterpreterState *interp); PyAPI_FUNC(void) _PyThreadState_Bind(PyThreadState *tstate); -// We keep this around exclusively for stable ABI compatibility. -PyAPI_FUNC(void) _PyThreadState_Init( - PyThreadState *tstate); PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate); extern void _PyThreadState_InitDetached(PyThreadState *, PyInterpreterState *); @@ -133,6 +130,18 @@ extern void _PyThreadState_ClearDetached(PyThreadState *); extern void _PyThreadState_BindDetached(PyThreadState *); extern void _PyThreadState_UnbindDetached(PyThreadState *); +PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); + +/* The implementation of sys._current_frames() Returns a dict mapping + thread id to that thread's current frame. +*/ +extern PyObject* _PyThread_CurrentFrames(void); + +/* The implementation of sys._current_exceptions() Returns a dict mapping + thread id to that thread's current exception. +*/ +extern PyObject* _PyThread_CurrentExceptions(void); + /* Other */ @@ -161,6 +170,10 @@ PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_UNLOCK(runtime) \ PyThread_release_lock((runtime)->interpreters.mutex) +// Get the configuration of the current interpreter. +// The caller must hold the GIL. +PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); + #ifdef __cplusplus } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ec2e64f2f46fc1..398450d804f5c5 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2714,11 +2714,6 @@ test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) assert(PyDict_Check(dict)); // dict is a borrowed reference - // private _PyThreadState_GetDict() - PyObject *dict2 = _PyThreadState_GetDict(tstate); - assert(dict2 == dict); - // dict2 is a borrowed reference - // PyThreadState_GetInterpreter() PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); assert(interp != NULL); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f6ae389ea05679..2e0609dcae5b7d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1234,6 +1234,27 @@ tracemalloc_get_traceback(PyObject *self, PyObject *args) } +// Test PyThreadState C API +static PyObject * +test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // PyThreadState_Get() + PyThreadState *tstate = PyThreadState_Get(); + assert(tstate != NULL); + + // test _PyThreadState_GetDict() + PyObject *dict = PyThreadState_GetDict(); + assert(dict != NULL); + // dict is a borrowed reference + + PyObject *dict2 = _PyThreadState_GetDict(tstate); + assert(dict2 == dict); + // dict2 is a borrowed reference + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1284,6 +1305,7 @@ static PyMethodDef module_functions[] = { {"_PyTime_ObjectToTimespec", test_pytime_object_to_timespec, METH_VARARGS}, {"_PyTime_ObjectToTimeval", test_pytime_object_to_timeval, METH_VARARGS}, {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS}, + {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 4801f37d6f6c5f..40dea170fd1f8b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1,8 +1,12 @@ - /* interpreters module */ /* low-level access to interpreter primitives */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "pycore_interp.h" // _PyInterpreterState_GetMainModule() #include "interpreteridobject.h" diff --git a/Python/compile.c b/Python/compile.c index d83bf0855ec257..29ea2742fad0cf 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -33,6 +33,7 @@ #include "pycore_compile.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_symtable.h" // PySTEntryObject, _PyFuture_FromAST() #include "opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed diff --git a/Python/frozenmain.c b/Python/frozenmain.c index f8be165f7671df..767f9804903a9e 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -1,7 +1,8 @@ /* Python interpreter main program for frozen scripts */ #include "Python.h" -#include "pycore_runtime.h" // _PyRuntime_Initialize() +#include "pycore_pystate.h" // _Py_GetConfig() +#include "pycore_runtime.h" // _PyRuntime_Initialize() #include #ifdef MS_WINDOWS diff --git a/Python/pystate.c b/Python/pystate.c index 20b02ef22109e7..50ce1d06602106 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1166,7 +1166,7 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) The GIL must be held. */ -PyInterpreterState * +PyInterpreterState* PyInterpreterState_Get(void) { PyThreadState *tstate = current_fast_get(&_PyRuntime); @@ -1408,7 +1408,7 @@ _PyThreadState_New(PyInterpreterState *interp) } // We keep this for stable ABI compabibility. -PyThreadState * +PyAPI_FUNC(PyThreadState*) _PyThreadState_Prealloc(PyInterpreterState *interp) { return _PyThreadState_New(interp); @@ -1416,7 +1416,7 @@ _PyThreadState_Prealloc(PyInterpreterState *interp) // We keep this around for (accidental) stable ABI compatibility. // Realistically, no extensions are using it. -void +PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *tstate) { Py_FatalError("_PyThreadState_Init() is for internal use only"); From d5bd32fb48ef8db2586b09d951514d75437b6195 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 2 Jul 2023 15:07:46 +0900 Subject: [PATCH 07/31] gh-104922: remove PY_SSIZE_T_CLEAN (#106315) --- Modules/_bisectmodule.c | 1 - Modules/_bz2module.c | 2 -- Modules/_codecsmodule.c | 1 - Modules/_ctypes/_ctypes.c | 1 - Modules/_cursesmodule.c | 2 -- Modules/_dbmmodule.c | 1 - Modules/_elementtree.c | 2 -- Modules/_gdbmmodule.c | 1 - Modules/_hashopenssl.c | 2 -- Modules/_io/_iomodule.c | 1 - Modules/_io/bufferedio.c | 1 - Modules/_io/fileio.c | 1 - Modules/_io/iobase.c | 1 - Modules/_io/stringio.c | 1 - Modules/_io/textio.c | 1 - Modules/_io/winconsoleio.c | 1 - Modules/_localemodule.c | 1 - Modules/_lzmamodule.c | 2 -- Modules/_multiprocessing/multiprocessing.h | 2 -- Modules/_multiprocessing/posixshmem.c | 2 -- Modules/_sqlite/connection.h | 1 - Modules/_sqlite/cursor.h | 1 - Modules/_sqlite/microprotocols.h | 1 - Modules/_sqlite/module.h | 1 - Modules/_sqlite/row.h | 1 - Modules/_sqlite/statement.h | 1 - Modules/_sqlite/util.h | 1 - Modules/_sre/sre.c | 2 -- Modules/_ssl.c | 2 -- Modules/_stat.c | 1 - Modules/_struct.c | 2 -- Modules/_testbuffer.c | 3 --- Modules/_testcapi/float.c | 2 -- Modules/_testcapi/getargs.c | 2 -- Modules/_testcapi/structmember.c | 1 - Modules/_testcapi/unicode.c | 1 - Modules/_testcapimodule.c | 2 -- Modules/_testclinic.c | 2 -- Modules/_testinternalcapi.c | 2 -- Modules/_tkinter.c | 1 - Modules/_uuidmodule.c | 2 -- Modules/_xxtestfuzz/_xxtestfuzz.c | 1 - Modules/arraymodule.c | 1 - Modules/binascii.c | 2 -- Modules/cjkcodecs/cjkcodecs.h | 1 - Modules/cjkcodecs/multibytecodec.c | 1 - Modules/fcntlmodule.c | 3 --- Modules/itertoolsmodule.c | 1 - Modules/mmapmodule.c | 1 - Modules/posixmodule.c | 2 -- Modules/socketmodule.c | 1 - Modules/unicodedata.c | 2 -- Modules/zlibmodule.c | 2 -- Objects/bytearrayobject.c | 1 - Objects/bytes_methods.c | 1 - Objects/bytesobject.c | 2 -- Objects/exceptions.c | 1 - Objects/fileobject.c | 1 - Objects/picklebufobject.c | 1 - Objects/unicodeobject.c | 1 - PC/winreg.c | 1 - Parser/pegen.h | 1 - Parser/tokenizer.c | 1 - Python/marshal.c | 2 -- 64 files changed, 90 deletions(-) diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 60f4dc69dc05d9..9e0fd336419b44 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -7,7 +7,6 @@ Converted to C by Dmitry Vasiliev (dima at hlabs.spb.ru). # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallMethod() diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 97bd44b4ac9694..eeefe6034998c8 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -1,7 +1,5 @@ /* _bz2 - Low-level Python interface to libbzip2. */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" // PyMemberDef diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 4dfd134f1cb11e..c31c1b6d6f2bbc 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -30,7 +30,6 @@ Copyright (c) Corporation for National Research Initiatives. ------------------------------------------------------------------------ */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_codecs.h" // _PyCodec_Lookup() diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 840d0df85e9c87..a9d8a2b9cb4ad7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -101,7 +101,6 @@ bytes(cdata) #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN #include "Python.h" // windows.h must be included before pycore internal headers diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 5691a419a32f8e..1f5afa6fcd898d 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -104,8 +104,6 @@ static const char PyCursesVersion[] = "2.2"; # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_structseq.h" // _PyStructSequence_NewType() diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 9908174c94c450..5be444d53e8da3 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -2,7 +2,6 @@ /* DBM module using dictionary interface */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2a0eac4d2f8085..48280690a707a4 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -11,8 +11,6 @@ *-------------------------------------------------------------------- */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" // PyMemberDef #include "expat.h" diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 4dbb5741b2ede8..bedbdc081425c2 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -3,7 +3,6 @@ /* Author: Anthony Baxter, after dbmmodule.c */ /* Doc strings: Mitch Chapman */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "gdbm.h" diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 259db809b8664c..011cb765ed82e6 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -22,8 +22,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_hashtable.h" #include "hashlib.h" diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 7b06c1bee5a832..1a7920eebe0b61 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -7,7 +7,6 @@ Mostly written by Amaury Forgeot d'Arc */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "_iomodule.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index f30d54a5e11b0a..25376f82042836 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -7,7 +7,6 @@ Written by Amaury Forgeot d'Arc and Antoine Pitrou */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 005c9ffe56f71a..1a5b61301dec58 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1,6 +1,5 @@ /* Author: Daniel Stutzbach */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 5cd679c68b1281..729a708a3fc40e 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -8,7 +8,6 @@ */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_long.h" // _PyLong_GetOne() diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index d4028451754cef..1960002d405edf 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,4 +1,3 @@ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include // offsetof() #include "pycore_object.h" diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index c8e6792657cd58..f704875280da32 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -6,7 +6,6 @@ Written by Amaury Forgeot d'Arc and Antoine Pitrou */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_codecs.h" // _PyCodecInfo_GetIncrementalDecoder() diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 15f3053957da61..452b12c138fa8b 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -6,7 +6,6 @@ Written by Steve Dower */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 1ada7305117bb7..970530facd01b0 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -9,7 +9,6 @@ This software comes with no warranty. Use at your own risk. ******************************************************************/ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index e34fbad230d51a..ba0987ee0abca1 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -5,8 +5,6 @@ */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" // PyMemberDef diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index dfc2a8e0799a60..47257fd5d9fb26 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -1,8 +1,6 @@ #ifndef MULTIPROCESSING_H #define MULTIPROCESSING_H -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" #include "pythread.h" diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 88c93fe313785c..debef3267f77d1 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,8 +2,6 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ -#define PY_SSIZE_T_CLEAN - #include // for shm_open() and shm_unlink() diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 1df92065a587a2..7a748ee3ea0c58 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -23,7 +23,6 @@ #ifndef PYSQLITE_CONNECTION_H #define PYSQLITE_CONNECTION_H -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pythread.h" #include "structmember.h" diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h index 0bcdddc3e29595..42f817af7c54ad 100644 --- a/Modules/_sqlite/cursor.h +++ b/Modules/_sqlite/cursor.h @@ -23,7 +23,6 @@ #ifndef PYSQLITE_CURSOR_H #define PYSQLITE_CURSOR_H -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "statement.h" diff --git a/Modules/_sqlite/microprotocols.h b/Modules/_sqlite/microprotocols.h index 6bde9d01f45299..8a8c33525ee53b 100644 --- a/Modules/_sqlite/microprotocols.h +++ b/Modules/_sqlite/microprotocols.h @@ -26,7 +26,6 @@ #ifndef PSYCOPG_MICROPROTOCOLS_H #define PSYCOPG_MICROPROTOCOLS_H 1 -#define PY_SSIZE_T_CLEAN #include /** exported functions **/ diff --git a/Modules/_sqlite/module.h b/Modules/_sqlite/module.h index daa22091d38ad7..a4ca45cf6326a9 100644 --- a/Modules/_sqlite/module.h +++ b/Modules/_sqlite/module.h @@ -23,7 +23,6 @@ #ifndef PYSQLITE_MODULE_H #define PYSQLITE_MODULE_H -#define PY_SSIZE_T_CLEAN #include "Python.h" #define LEGACY_TRANSACTION_CONTROL -1 diff --git a/Modules/_sqlite/row.h b/Modules/_sqlite/row.h index b51909817584ba..d42b781e493177 100644 --- a/Modules/_sqlite/row.h +++ b/Modules/_sqlite/row.h @@ -23,7 +23,6 @@ #ifndef PYSQLITE_ROW_H #define PYSQLITE_ROW_H -#define PY_SSIZE_T_CLEAN #include "Python.h" typedef struct _Row diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index 11a6464b1a1c2b..b18f170ebb0708 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -23,7 +23,6 @@ #ifndef PYSQLITE_STATEMENT_H #define PYSQLITE_STATEMENT_H -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "connection.h" diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index a22bcd82d2a05b..68b1a8cb67ace3 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -23,7 +23,6 @@ #ifndef PYSQLITE_UTIL_H #define PYSQLITE_UTIL_H -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pythread.h" #include "sqlite3.h" diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 328e4be2fb5e43..3f11916f9e1726 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -38,8 +38,6 @@ static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 4254fde0f5190b..df1496925f678e 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -25,8 +25,6 @@ #endif #define OPENSSL_NO_DEPRECATED 1 -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() diff --git a/Modules/_stat.c b/Modules/_stat.c index 4218799103b59d..9747d848cbacb8 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,7 +11,6 @@ * */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #ifdef __cplusplus diff --git a/Modules/_struct.c b/Modules/_struct.c index 4f9478bd98095d..0a6f076aac0c53 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -7,8 +7,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 63ed4dc6ca80e1..5101834cfe1387 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1,9 +1,6 @@ /* C Extension module to test all aspects of PEP-3118. Written by Stefan Krah. */ - -#define PY_SSIZE_T_CLEAN - #include "Python.h" diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index 33cbda83a81af7..cff53fb950fcc6 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -1,5 +1,3 @@ -#define PY_SSIZE_T_CLEAN - #include "parts.h" #include "clinic/float.c.h" diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index a473c41f60af3c..10a1c1dd05253d 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -3,8 +3,6 @@ * APIs that parse and build arguments. */ -#define PY_SSIZE_T_CLEAN - #include "parts.h" static PyObject * diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index 0fb872a4328d60..8522dc962efa40 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -1,4 +1,3 @@ -#define PY_SSIZE_T_CLEAN #include "parts.h" #include // for offsetof() diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 73929eaffc676d..367d2af1413e57 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1,6 +1,5 @@ #include // ptrdiff_t -#define PY_SSIZE_T_CLEAN #include "parts.h" static struct PyModuleDef *_testcapimodule = NULL; // set at initialization diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 398450d804f5c5..ce1131743eb2a4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -17,8 +17,6 @@ /* Always enable assertions */ #undef NDEBUG -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "frameobject.h" // PyFrame_New #include "marshal.h" // PyMarshal_WriteLongToFile diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 6ff55a2755cf5a..26cdb4371ca24c 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -5,8 +5,6 @@ /* Always enable assertions */ #undef NDEBUG -#define PY_SSIZE_T_CLEAN - #include "Python.h" diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 2e0609dcae5b7d..971b8efb5fd9a4 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -9,8 +9,6 @@ /* Always enable assertions */ #undef NDEBUG -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "frameobject.h" #include "interpreteridobject.h" // _PyInterpreterID_LookUp() diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 97e5b2f738aa2a..76af803bd6eefb 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -21,7 +21,6 @@ Copyright (C) 1994 Steen Lumholt. */ -#define PY_SSIZE_T_CLEAN #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index ed3b2fedfd4d88..2f5be1c5144d83 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,8 +3,6 @@ * DCE compatible Universally Unique Identifier library. */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #if defined(HAVE_UUID_H) // AIX, FreeBSD, libuuid with pkgconf diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index e0694de6719df0..a2dbabce71ed67 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -1,4 +1,3 @@ -#define PY_SSIZE_T_CLEAN #include #include #include diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 15f7766a8dd284..f43a23467976b8 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -7,7 +7,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_moduleobject.h" // _PyModule_GetState() diff --git a/Modules/binascii.c b/Modules/binascii.c index 356947d43e43a9..cf9328795c2bcc 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -57,8 +57,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_long.h" // _PyLong_DigitValue #include "pycore_strhex.h" // _Py_strhex_bytes_with_sep() diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 36bc7024df9acc..48cdcfb3ad3087 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -7,7 +7,6 @@ #ifndef _CJKCODECS_H_ #define _CJKCODECS_H_ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "multibytecodec.h" diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 1070a751ffe638..cf437d09f1fe8d 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -4,7 +4,6 @@ * Written by Hye-Shik Chang */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "structmember.h" // PyMemberDef #include "multibytecodec.h" diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 6ca0b62bc5dca8..e530621fd269a1 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,8 +1,5 @@ - /* fcntl module */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #ifdef HAVE_SYS_FILE_H diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ae63bae79d5d07..13ff253f560b04 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,4 +1,3 @@ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6bde9939eaa2ca..fef27123a3282b 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -22,7 +22,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN #include #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d73886f14cb9ec..b9f42476e6b82c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7,8 +7,6 @@ of the compiler used. Different compilers define their own feature test macro, e.g. '_MSC_VER'. */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #ifdef __VXWORKS__ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 3add80233522b9..1d3f34b857a1d2 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -105,7 +105,6 @@ Local naming conventions: # pragma weak inet_aton #endif -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 7be4d83c6f070f..b6e50528a23c86 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -16,8 +16,6 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "structmember.h" // PyMemberDef diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 534d065765f0f9..c0f6b96f51baba 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -3,8 +3,6 @@ /* Windows users: read Python's PCbuild\readme.txt */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" // PyMemberDef #include "zlib.h" diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 944ec65b64f6a2..18a24a369a64c1 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1,6 +1,5 @@ /* PyByteArray (bytearray) implementation */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 33aa9c3db6e805..c1bc6383df30ce 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -1,4 +1,3 @@ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index bf54ec1d51d209..477bc4d31e812b 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1,7 +1,5 @@ /* bytes object implementation */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytesobject.h" // _PyBytes_Find(), _PyBytes_Repeat() diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 2bfa07ea3121f4..85cf2cca16516b 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -4,7 +4,6 @@ * Thanks go to Tim Peters and Michael Hudson for debugging. */ -#define PY_SSIZE_T_CLEAN #include #include #include "pycore_abstract.h" // _PyObject_RealIsSubclass() diff --git a/Objects/fileobject.c b/Objects/fileobject.c index e99e155f2b8c98..6d980a1b379166 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -1,6 +1,5 @@ /* File object implementation (what's left of it -- see io.py) */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_runtime.h" // _PyRuntime diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index aaa852cfbb05b0..ca83a0a0806ce1 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -1,6 +1,5 @@ /* PickleBuffer object implementation */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 74def5ada5134e..79402714f23b9d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -38,7 +38,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_atomic_funcs.h" // _Py_atomic_size_get() diff --git a/PC/winreg.c b/PC/winreg.c index 279d48f792b96a..aa2055c7e619d2 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -12,7 +12,6 @@ */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_object.h" // _PyObject_Init() #include "pycore_moduleobject.h" diff --git a/Parser/pegen.h b/Parser/pegen.h index fe13d10e6b83e3..5f29285951e812 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -1,7 +1,6 @@ #ifndef PEGEN_H #define PEGEN_H -#define PY_SSIZE_T_CLEAN #include #include #include diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 6bdf371a7adf03..f19198600fa018 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1,7 +1,6 @@ /* Tokenizer implementation */ -#define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() diff --git a/Python/marshal.c b/Python/marshal.c index 7cfc7cc00306f5..517220a4463cf3 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -6,8 +6,6 @@ Version 3 of this protocol properly supports circular links and sharing. */ -#define PY_SSIZE_T_CLEAN - #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_code.h" // _PyCode_New() From 5950e7dbfcd9307c7e74184a1586ef99f9f35c3b Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 2 Jul 2023 17:15:28 +0300 Subject: [PATCH 08/31] gh-106217: Truncate the issue body size of `new-bugs-announce-notifier` (#106329) Co-authored-by: Hugo van Kemenade --- .github/workflows/new-bugs-announce-notifier.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 73806c5d6d58af..e3572db670693e 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -41,7 +41,10 @@ jobs: url : issue.data.html_url, labels : issue.data.labels.map(label => { return label.name }).join(", "), assignee : issue.data.assignees.map(assignee => { return assignee.login }), - body : issue.data.body + // We need to truncate the body size, because the max size for + // the whole payload is 16kb. We want to be safe and assume that + // body can take up to ~8kb of space. + body : issue.data.body.substring(8000) }; const data = { From 20b7c79b9d67a761aaf818d3b92498ea0b0d80d9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 3 Jul 2023 00:54:35 +0900 Subject: [PATCH 09/31] gh-104922: Doc: add note about PY_SSIZE_T_CLEAN (#106314) Add note about PY_SSIZE_T_CLEAN in extending and embedding document. --- Doc/extending/embedding.rst | 7 +++++++ Doc/extending/extending.rst | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 1470029b804779..bd1abe36cbb80e 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -87,6 +87,13 @@ perform some operation on a file. :: Py_ExitStatusException(status); } +.. note:: + + ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be + used in some APIs instead of ``int``. + It is not necessary since Python 3.13, but we keep it here for backward compatibility. + See :ref:`arg-parsing-string-and-buffers` for a description of this macro. + Setting :c:member:`PyConfig.program_name` should be called before :c:func:`Py_InitializeFromConfig` to inform the interpreter about paths to Python run-time libraries. Next, the Python interpreter is initialized with diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index ef93848840861c..7d08bb9f6b8dd8 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -69,8 +69,10 @@ the module and a copyright notice if you like). headers on some systems, you *must* include :file:`Python.h` before any standard headers are included. - It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including - ``Python.h``. See :ref:`arg-parsing-string-and-buffers` for a description of this macro. + ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be + used in some APIs instead of ``int``. + It is not necessary since Python 3.13, but we keep it here for backward compatibility. + See :ref:`arg-parsing-string-and-buffers` for a description of this macro. All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or ``PY``, except those defined in standard header files. For convenience, and From 9a51a419619bb9dd1075d683708c57803c5d48c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 18:11:45 +0200 Subject: [PATCH 10/31] gh-106320: Remove private _PyInterpreterState functions (#106335) Remove private _PyThreadState and _PyInterpreterState C API functions: move them to the internal C API (pycore_pystate.h and pycore_interp.h). Don't export most of these functions anymore, but still export functions used by tests. Remove _PyThreadState_Prealloc() and _PyThreadState_Init() from the C API, but keep it in the stable API. From bc7eb1708452da59c22782c487ae7f05f1788970 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 18:37:37 +0200 Subject: [PATCH 11/31] gh-106320: Use _PyInterpreterState_GET() (#106336) Replace PyInterpreterState_Get() with inlined _PyInterpreterState_GET(). --- Modules/_asynciomodule.c | 2 +- Modules/_io/bufferedio.c | 2 +- Modules/_posixsubprocess.c | 2 +- Modules/_testinternalcapi.c | 6 +++--- Modules/_typingmodule.c | 7 ++++--- Modules/_winapi.c | 3 ++- Objects/typevarobject.c | 22 +++++++++++----------- Python/codecs.c | 4 ++-- Python/instrumentation.c | 10 ++++------ Python/optimizer.c | 9 ++++----- Python/pystate.c | 2 +- 11 files changed, 34 insertions(+), 35 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 5b28f2dd28a221..05f94ef9ed2816 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -527,7 +527,7 @@ future_init(FutureObj *fut, PyObject *loop) if (is_true < 0) { return -1; } - if (is_true && !_Py_IsInterpreterFinalizing(PyInterpreterState_Get())) { + if (is_true && !_Py_IsInterpreterFinalizing(_PyInterpreterState_GET())) { /* Only try to capture the traceback if the interpreter is not being finalized. The original motivation to add a `_Py_IsFinalizing()` call was to prevent SIGSEGV when a Future is created in a __del__ diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 25376f82042836..9c6a9cddba2258 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -292,7 +292,7 @@ _enter_buffered_busy(buffered *self) "reentrant call inside %R", self); return 0; } - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); relax_locking = _Py_IsInterpreterFinalizing(interp); Py_BEGIN_ALLOW_THREADS if (!relax_locking) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 52e1d2faf3e9fa..6caa4b8852911e 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1025,7 +1025,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, int *c_fds_to_keep = NULL; Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if ((preexec_fn != Py_None) && interp->finalizing) { PyErr_SetString(PyExc_RuntimeError, "preexec_fn not supported at interpreter shutdown"); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 971b8efb5fd9a4..4875ee7bed1683 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -559,7 +559,7 @@ static PyObject * set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args)) { module_state *state = get_module_state(self); - _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), _PyEval_EvalFrameDefault); + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), _PyEval_EvalFrameDefault); Py_CLEAR(state->record_list); Py_RETURN_NONE; } @@ -587,7 +587,7 @@ set_eval_frame_record(PyObject *self, PyObject *list) return NULL; } Py_XSETREF(state->record_list, Py_NewRef(list)); - _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), record_eval); + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), record_eval); Py_RETURN_NONE; } @@ -883,7 +883,7 @@ pending_threadfunc(PyObject *self, PyObject *args, PyObject *kwargs) { return NULL; } - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); /* create the reference for the callbackwhile we hold the lock */ Py_INCREF(callable); diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 39a124a26adf31..59d3a80a9305db 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -5,8 +5,9 @@ #endif #include "Python.h" -#include "internal/pycore_interp.h" -#include "internal/pycore_typevarobject.h" +#include "pycore_interp.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_typevarobject.h" #include "clinic/_typingmodule.c.h" /*[clinic input] @@ -44,7 +45,7 @@ PyDoc_STRVAR(typing_doc, static int _typing_exec(PyObject *m) { - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); #define EXPORT_TYPE(name, typename) \ if (PyModule_AddObjectRef(m, name, \ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index a7e6bb582fc64d..d4291f557b6a66 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -36,6 +36,7 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pystate.h" // _PyInterpreterState_GET #include "structmember.h" // PyMemberDef @@ -133,7 +134,7 @@ overlapped_dealloc(OverlappedObject *self) { /* The operation is no longer pending -- nothing to do. */ } - else if (_Py_IsInterpreterFinalizing(PyInterpreterState_Get())) + else if (_Py_IsInterpreterFinalizing(_PyInterpreterState_GET())) { /* The operation is still pending -- give a warning. This will probably only happen on Windows XP. */ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 406a6eb76e3a8a..0b44f84b6f01d2 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1,6 +1,6 @@ // TypeVar, TypeVarTuple, and ParamSpec #include "Python.h" -#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK +#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK #include "pycore_typevarobject.h" #include "pycore_unionobject.h" // _Py_union_type_or #include "structmember.h" @@ -144,7 +144,7 @@ static int contains_typevartuple(PyTupleObject *params) { Py_ssize_t n = PyTuple_GET_SIZE(params); - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.typevartuple_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; for (Py_ssize_t i = 0; i < n; i++) { PyObject *param = PyTuple_GET_ITEM(params, i); if (Py_IS_TYPE(param, tp)) { @@ -165,7 +165,7 @@ unpack_typevartuples(PyObject *params) if (new_params == NULL) { return NULL; } - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.typevartuple_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; for (Py_ssize_t i = 0; i < n; i++) { PyObject *param = PyTuple_GET_ITEM(params, i); if (Py_IS_TYPE(param, tp)) { @@ -291,7 +291,7 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.typevar_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevar_type; assert(tp != NULL); typevarobject *tv = PyObject_GC_New(typevarobject, tp); if (tv == NULL) { @@ -576,7 +576,7 @@ paramspecargs_repr(PyObject *self) { paramspecattrobject *psa = (paramspecattrobject *)self; - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.paramspec_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; if (Py_IS_TYPE(psa->__origin__, tp)) { return PyUnicode_FromFormat("%U.args", ((paramspecobject *)psa->__origin__)->name); @@ -656,7 +656,7 @@ paramspeckwargs_repr(PyObject *self) { paramspecattrobject *psk = (paramspecattrobject *)self; - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.paramspec_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; if (Py_IS_TYPE(psk->__origin__, tp)) { return PyUnicode_FromFormat("%U.kwargs", ((paramspecobject *)psk->__origin__)->name); @@ -789,14 +789,14 @@ static PyMemberDef paramspec_members[] = { static PyObject * paramspec_args(PyObject *self, void *unused) { - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.paramspecargs_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspecargs_type; return (PyObject *)paramspecattr_new(tp, self); } static PyObject * paramspec_kwargs(PyObject *self, void *unused) { - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.paramspeckwargs_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspeckwargs_type; return (PyObject *)paramspecattr_new(tp, self); } @@ -810,7 +810,7 @@ static paramspecobject * paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.paramspec_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; paramspecobject *ps = PyObject_GC_New(paramspecobject, tp); if (ps == NULL) { return NULL; @@ -1059,7 +1059,7 @@ static PyMemberDef typevartuple_members[] = { static typevartupleobject * typevartuple_alloc(PyObject *name, PyObject *module) { - PyTypeObject *tp = PyInterpreterState_Get()->cached_objects.typevartuple_type; + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); if (tvt == NULL) { return NULL; @@ -1598,7 +1598,7 @@ _Py_subscript_generic(PyThreadState* unused, PyObject *params) { params = unpack_typevartuples(params); - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->cached_objects.generic_type == NULL) { PyErr_SetString(PyExc_SystemError, "Cannot find Generic type"); return NULL; diff --git a/Python/codecs.c b/Python/codecs.c index 1983f56ba204c1..f9f23005debb1f 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -11,7 +11,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.codec_search_path -#include "pycore_pyerrors.h" // _PyErr_FormatNote() +#include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include @@ -55,7 +55,7 @@ int PyCodec_Register(PyObject *search_function) int PyCodec_Unregister(PyObject *search_function) { - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *codec_search_path = interp->codec_search_path; /* Do nothing if codec_search_path is not created yet or was cleared. */ if (codec_search_path == NULL) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3253a0ea13033c..03d7d2f215af7c 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1,5 +1,3 @@ - - #include "Python.h" #include "pycore_call.h" #include "pycore_frame.h" @@ -9,7 +7,7 @@ #include "pycore_object.h" #include "pycore_opcode.h" #include "pycore_pyerrors.h" -#include "pycore_pystate.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() /* Uncomment this to dump debugging output when assertions fail */ // #define INSTRUMENT_DEBUG 1 @@ -390,7 +388,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) fprintf(out, "NULL\n"); return; } - dump_monitors("Global", PyInterpreterState_Get()->monitors, out); + dump_monitors("Global", _PyInterpreterState_GET()->monitors, out); dump_monitors("Code", data->local_monitors, out); dump_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); @@ -449,7 +447,7 @@ sanity_check_instrumentation(PyCodeObject *code) if (data == NULL) { return; } - _Py_Monitors active_monitors = PyInterpreterState_Get()->monitors; + _Py_Monitors active_monitors = _PyInterpreterState_GET()->monitors; if (code->_co_monitoring) { _Py_Monitors local_monitors = code->_co_monitoring->local_monitors; active_monitors = monitors_or(active_monitors, local_monitors); @@ -740,7 +738,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) static bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) { - int global_tools = PyInterpreterState_Get()->monitors.tools[event]; + int global_tools = _PyInterpreterState_GET()->monitors.tools[event]; int local_tools = code->_co_monitoring->local_monitors.tools[event]; return tools == ((global_tools | local_tools) & tools); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 9d77ab4ff879bb..b00825ade16e07 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,10 +1,9 @@ - #include "Python.h" #include "opcode.h" #include "pycore_interp.h" #include "pycore_opcode.h" #include "opcode_metadata.h" -#include "pycore_pystate.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uops.h" #include "cpython/optimizer.h" #include @@ -125,7 +124,7 @@ _PyOptimizerObject _PyOptimizer_Default = { _PyOptimizerObject * PyUnstable_GetOptimizer(void) { - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->optimizer == &_PyOptimizer_Default) { return NULL; } @@ -138,7 +137,7 @@ PyUnstable_GetOptimizer(void) void PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) { - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; } @@ -155,7 +154,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI { PyCodeObject *code = (PyCodeObject *)frame->f_executable; assert(PyCode_Check(code)); - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!has_space_for_executor(code, src)) { goto jump_to_destination; } diff --git a/Python/pystate.c b/Python/pystate.c index 50ce1d06602106..a9b404bd5c93e3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2835,7 +2835,7 @@ _PyInterpreterState_GetConfig(PyInterpreterState *interp) int _PyInterpreterState_GetConfigCopy(PyConfig *config) { - PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); PyStatus status = _PyConfig_Copy(config, &interp->config); if (PyStatus_Exception(status)) { From dbefa88b27ee1f124f782363b139aee3f1ccf590 Mon Sep 17 00:00:00 2001 From: Charlie Zhao Date: Mon, 3 Jul 2023 00:50:40 +0800 Subject: [PATCH 12/31] gh-106078: Move DecimalException to _decimal state (#106301) --- Modules/_decimal/_decimal.c | 17 +++++++++-------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index b7cb19515b3002..da623725003428 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -46,6 +46,9 @@ typedef struct { PyTypeObject *PyDec_Type; PyTypeObject *PyDecSignalDict_Type; PyTypeObject *DecimalTuple; + + /* Top level Exception; inherits from ArithmeticError */ + PyObject *DecimalException; } decimal_state; static decimal_state global_state; @@ -164,9 +167,6 @@ typedef struct { PyObject *ex; /* corresponding exception */ } DecCondMap; -/* Top level Exception; inherits from ArithmeticError */ -static PyObject *DecimalException = NULL; - /* Exceptions that correspond to IEEE signals */ #define SUBNORMAL 5 #define INEXACT 6 @@ -5902,10 +5902,10 @@ PyInit__decimal(void) CHECK_INT(PyModule_AddType(m, state->DecimalTuple)); /* Create top level exception */ - ASSIGN_PTR(DecimalException, PyErr_NewException( + ASSIGN_PTR(state->DecimalException, PyErr_NewException( "decimal.DecimalException", PyExc_ArithmeticError, NULL)); - CHECK_INT(PyModule_AddObject(m, "DecimalException", Py_NewRef(DecimalException))); + CHECK_INT(PyModule_AddType(m, (PyTypeObject *)state->DecimalException)); /* Create signal tuple */ ASSIGN_PTR(SignalTuple, PyTuple_New(SIGNAL_MAP_LEN)); @@ -5918,10 +5918,11 @@ PyInit__decimal(void) switch (cm->flag) { case MPD_Float_operation: - base = PyTuple_Pack(2, DecimalException, PyExc_TypeError); + base = PyTuple_Pack(2, state->DecimalException, PyExc_TypeError); break; case MPD_Division_by_zero: - base = PyTuple_Pack(2, DecimalException, PyExc_ZeroDivisionError); + base = PyTuple_Pack(2, state->DecimalException, + PyExc_ZeroDivisionError); break; case MPD_Overflow: base = PyTuple_Pack(2, signal_map[INEXACT].ex, @@ -5933,7 +5934,7 @@ PyInit__decimal(void) signal_map[SUBNORMAL].ex); break; default: - base = PyTuple_Pack(1, DecimalException); + base = PyTuple_Pack(1, state->DecimalException); break; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 1131edff265ee4..8fdc54df2b0722 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -393,7 +393,6 @@ Modules/xxlimited_35.c - Xxo_Type - ## exception types Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - -Modules/_decimal/_decimal.c - DecimalException - Modules/_tkinter.c - Tkinter_TclError - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - From c38c66687f148991031914f12ebd5c42d6e26b5b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 23:19:59 +0200 Subject: [PATCH 13/31] gh-106320: Add pycore_complexobject.h header file (#106339) Add internal pycore_complexobject.h header file. Move _Py_c_xxx() functions and _PyComplex_FormatAdvancedWriter() function to this new header file. --- Include/cpython/complexobject.h | 21 ---------------- Include/internal/pycore_complexobject.h | 33 +++++++++++++++++++++++++ Makefile.pre.in | 1 + Modules/cmathmodule.c | 1 + Objects/complexobject.c | 1 + Objects/stringlib/unicode_format.h | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 8 files changed, 41 insertions(+), 21 deletions(-) create mode 100644 Include/internal/pycore_complexobject.h diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index b7d7283ae88965..b524ec42c24371 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,16 +7,6 @@ typedef struct { double imag; } Py_complex; -/* Operations on complex numbers from complexmodule.c */ - -PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); -PyAPI_FUNC(double) _Py_c_abs(Py_complex); - /* Complex object interface */ /* @@ -31,14 +21,3 @@ typedef struct { PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex); PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *op); - -#ifdef Py_BUILD_CORE -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -extern int _PyComplex_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif // Py_BUILD_CORE diff --git a/Include/internal/pycore_complexobject.h b/Include/internal/pycore_complexobject.h new file mode 100644 index 00000000000000..fb344b7bb79bd8 --- /dev/null +++ b/Include/internal/pycore_complexobject.h @@ -0,0 +1,33 @@ +#ifndef Py_INTERNAL_COMPLEXOBJECT_H +#define Py_INTERNAL_COMPLEXOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* Operations on complex numbers from complexmodule.c */ + +PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); +PyAPI_FUNC(double) _Py_c_abs(Py_complex); + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +extern int _PyComplex_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_COMPLEXOBJECT_H diff --git a/Makefile.pre.in b/Makefile.pre.in index e788e590dcbb43..7560d17e3796f0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1734,6 +1734,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_code.h \ $(srcdir)/Include/internal/pycore_codecs.h \ $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_complexobject.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_dict.h \ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 1a31bdc824bb03..db8b321e72e8ce 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -7,6 +7,7 @@ #endif #include "Python.h" +#include "pycore_complexobject.h" // _Py_c_neg() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR /* we need DBL_MAX, DBL_MIN, DBL_EPSILON, DBL_MANT_DIG and FLT_RADIX from float.h. We assume that FLT_RADIX is either 2 or 16. */ diff --git a/Objects/complexobject.c b/Objects/complexobject.c index aee03ddfb07aec..12968a63cd6fdd 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -7,6 +7,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _Py_ADJUST_ERANGE2() diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index f4ba0a92776a97..91c71ab5736c25 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -2,6 +2,7 @@ unicode_format.h -- implementation of str.format(). */ +#include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() /************************************************************************/ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 436381c3ea5db4..79ce2d3d14017e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -212,6 +212,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4d9acf4893872d..d47a22909e1e3a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -543,6 +543,9 @@ Include\internal + + Include\internal + Include\internal From 4e3aa7cd312759922556ee96aa42033c375027e2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Jul 2023 23:56:58 +0200 Subject: [PATCH 14/31] gh-106320: _testcapi avoids private _PyUnicode_EqualToASCIIString() (#106341) Replace private _PyUnicode_EqualToASCIIString() with public PyUnicode_CompareWithASCIIString(). --- Modules/_testcapi/unicode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 367d2af1413e57..9c2760c3f763a6 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1100,7 +1100,7 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) } \ else if (result == NULL) \ return NULL; \ - else if (!_PyUnicode_EqualToASCIIString(result, EXPECTED)) { \ + else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ PyErr_Format(PyExc_AssertionError, \ "test_string_from_format: failed at \"%s\" " \ "expected \"%s\" got \"%s\"", \ From 569887aa9da0da2eb2e2ad7d6a7496290f223ab0 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jul 2023 01:42:38 +0200 Subject: [PATCH 15/31] gh-104050: Add more type hints to Argument Clinic DSLParser() (#106343) Annotate the following method signatures: - state_dsl_start() - state_parameter_docstring_start() - state_parameters_start() Inverting ignore_line() logic, add type hints (including type guard) to it, and rename to valid_line(). --- Tools/clinic/clinic.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 43b7a9d9ac4aa7..27d3d572868883 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -29,7 +29,15 @@ from collections.abc import Callable from types import FunctionType, NoneType -from typing import Any, Final, NamedTuple, NoReturn, Literal, overload +from typing import ( + Any, + Final, + Literal, + NamedTuple, + NoReturn, + TypeGuard, + overload, +) # TODO: # @@ -4471,17 +4479,20 @@ def parse(self, block: Block) -> None: block.output = self.saved_output @staticmethod - def ignore_line(line): + def valid_line(line: str | None) -> TypeGuard[str]: + if line is None: + return False + # ignore comment-only lines if line.lstrip().startswith('#'): - return True + return False # Ignore empty lines too # (but not in docstring sections!) if not line.strip(): - return True + return False - return False + return True @staticmethod def calculate_indent(line: str) -> int: @@ -4497,9 +4508,9 @@ def next( if line is not None: self.state(line) - def state_dsl_start(self, line): + def state_dsl_start(self, line: str | None) -> None: # self.block = self.ClinicOutputBlock(self) - if self.ignore_line(line): + if not self.valid_line(line): return # is it a directive? @@ -4716,8 +4727,8 @@ def state_modulename_name(self, line): ps_start, ps_left_square_before, ps_group_before, ps_required, \ ps_optional, ps_group_after, ps_right_square_after = range(7) - def state_parameters_start(self, line): - if self.ignore_line(line): + def state_parameters_start(self, line: str) -> None: + if not self.valid_line(line): return # if this line is not indented, we have no parameters @@ -4742,7 +4753,7 @@ def state_parameter(self, line): line = self.parameter_continuation + ' ' + line.lstrip() self.parameter_continuation = '' - if self.ignore_line(line): + if not self.valid_line(line): return assert self.indent.depth == 2 @@ -5075,7 +5086,7 @@ def parse_special_symbol(self, symbol): fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY - def state_parameter_docstring_start(self, line): + def state_parameter_docstring_start(self, line: str) -> None: self.parameter_docstring_indent = len(self.indent.margin) assert self.indent.depth == 3 return self.next(self.state_parameter_docstring, line) From 987b712b4aeeece336eed24fcc87a950a756c3e2 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 2 Jul 2023 20:22:40 -0700 Subject: [PATCH 16/31] Replace the esoteric term 'datum' when describing dict comprehensions (#106119) --- Doc/reference/expressions.rst | 22 +++++++++++----------- Doc/reference/simple_stmts.rst | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 0c700f908d6878..ce1c9a59d58353 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -298,27 +298,27 @@ Dictionary displays .. index:: pair: dictionary; display pair: dictionary; comprehensions - key, datum, key/datum pair + key, value, key/value pair pair: object; dictionary single: {} (curly brackets); dictionary expression single: : (colon); in dictionary expressions single: , (comma); in dictionary displays -A dictionary display is a possibly empty series of key/datum pairs enclosed in -curly braces: +A dictionary display is a possibly empty series of dict items (key/value pairs) +enclosed in curly braces: .. productionlist:: python-grammar - dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" - key_datum_list: `key_datum` ("," `key_datum`)* [","] - key_datum: `expression` ":" `expression` | "**" `or_expr` + dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" + dict_item_list: `dict_item` ("," `dict_item`)* [","] + dict_item: `expression` ":" `expression` | "**" `or_expr` dict_comprehension: `expression` ":" `expression` `comp_for` A dictionary display yields a new dictionary object. -If a comma-separated sequence of key/datum pairs is given, they are evaluated +If a comma-separated sequence of dict items is given, they are evaluated from left to right to define the entries of the dictionary: each key object is -used as a key into the dictionary to store the corresponding datum. This means -that you can specify the same key multiple times in the key/datum list, and the +used as a key into the dictionary to store the corresponding value. This means +that you can specify the same key multiple times in the dict item list, and the final dictionary's value for that key will be the last one given. .. index:: @@ -328,7 +328,7 @@ final dictionary's value for that key will be the last one given. A double asterisk ``**`` denotes :dfn:`dictionary unpacking`. Its operand must be a :term:`mapping`. Each mapping item is added to the new dictionary. Later values replace values already set by -earlier key/datum pairs and earlier dictionary unpackings. +earlier dict items and earlier dictionary unpackings. .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. @@ -344,7 +344,7 @@ in the new dictionary in the order they are produced. Restrictions on the types of the key values are listed earlier in section :ref:`types`. (To summarize, the key type should be :term:`hashable`, which excludes all mutable objects.) Clashes between duplicate keys are not detected; the last -datum (textually rightmost in the display) stored for a given key value +value (textually rightmost in the display) stored for a given key value prevails. .. versionchanged:: 3.8 diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 662a4b643c4378..a9e65be1eda340 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -210,7 +210,7 @@ Assignment of an object to a single target is recursively defined as follows. If the primary is a mapping object (such as a dictionary), the subscript must have a type compatible with the mapping's key type, and the mapping is then - asked to create a key/datum pair which maps the subscript to the assigned + asked to create a key/value pair which maps the subscript to the assigned object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). From 0355625d94a50f4b816770bad946420d005900b8 Mon Sep 17 00:00:00 2001 From: Jeremy Paige Date: Sun, 2 Jul 2023 22:44:37 -0700 Subject: [PATCH 17/31] Document PYTHONSAFEPATH along side -P (#106122) --- Python/initconfig.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 147cb370412cd6..787e583262406c 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -49,7 +49,7 @@ Options (and corresponding environment variables):\n\ .pyc extension; also PYTHONOPTIMIZE=x\n\ -OO : do -O changes and also discard docstrings; add .opt-2 before\n\ .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path\n\ +-P : don't prepend a potentially unsafe path to sys.path; also PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ @@ -144,7 +144,6 @@ static const char usage_envvars[] = "PYTHONSTARTUP: file executed on interactive startup (no default)\n" "PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" " default module search path. The result is sys.path.\n" -"PYTHONSAFEPATH: don't prepend a potentially unsafe path to sys.path.\n" "PYTHONHOME : alternate directory (or %lc).\n" " The default module search path uses %s.\n" "PYTHONPLATLIBDIR : override sys.platlibdir.\n" @@ -184,6 +183,7 @@ static const char usage_envvars[] = " (-X int_max_str_digits=number)\n" "PYTHONNOUSERSITE : disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n" "PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" "PYTHONVERBOSE : trace import statements (-v)\n" "PYTHONWARNINGS=arg : warning control (-W arg)\n"; From d65b783b6966d233467a48ef633afb4aff9d5df8 Mon Sep 17 00:00:00 2001 From: Mario Corchero Date: Mon, 3 Jul 2023 07:56:54 +0100 Subject: [PATCH 18/31] gh-61215: New mock to wait for multi-threaded events to happen (#16094) mock: Add `ThreadingMock` class Add a new class that allows to wait for a call to happen by using `Event` objects. This mock class can be used to test and validate expectations of multithreading code. It uses two attributes for events to distinguish calls with any argument and calls with specific arguments. The calls with specific arguments need a lock to prevent two calls in parallel from creating the same event twice. The timeout is configured at class and constructor level to allow users to set a timeout, we considered passing it as an argument to the function but it could collide with a function parameter. Alternatively we also considered passing it as positional only but from an API caller perspective it was unclear what the first number meant on the function call, think `mock.wait_until_called(1, "arg1", "arg2")`, where 1 is the timeout. Lastly we also considered adding the new attributes to magic mock directly rather than having a custom mock class for multi threading scenarios, but we preferred to have specialised class that can be composed if necessary. Additionally, having added it to `MagicMock` directly would have resulted in `AsyncMock` having this logic, which would not work as expected, since when if user "waits" on a coroutine does not have the same meaning as waiting on a standard call. Co-authored-by: Karthikeyan Singaravelan --- Doc/library/unittest.mock.rst | 47 +++ .../testmock/testthreadingmock.py | 278 ++++++++++++++++++ Lib/unittest/mock.py | 94 ++++++ .../2019-09-13-13-28-10.bpo-17013.NWcgE3.rst | 3 + 4 files changed, 422 insertions(+) create mode 100644 Lib/test/test_unittest/testmock/testthreadingmock.py create mode 100644 Misc/NEWS.d/next/Library/2019-09-13-13-28-10.bpo-17013.NWcgE3.rst diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 6c4d801f69f5a9..756422bd6eab9e 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -205,8 +205,10 @@ The Mock Class import asyncio import inspect import unittest + import threading from unittest.mock import sentinel, DEFAULT, ANY from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock + from unittest.mock import ThreadingMock from unittest.mock import mock_open :class:`Mock` is a flexible mock object intended to replace the use of stubs and @@ -1099,6 +1101,51 @@ object:: [call('foo'), call('bar')] +.. class:: ThreadingMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, *, timeout=UNSET, **kwargs) + + A version of :class:`MagicMock` for multithreading tests. The + :class:`ThreadingMock` object provides extra methods to wait for a call to + be invoked, rather than assert on it immediately. + + The default timeout is specified by the ``timeout`` argument, or if unset by the + :attr:`ThreadingMock.DEFAULT_TIMEOUT` attribute, which defaults to blocking (``None``). + + You can configure the global default timeout by setting :attr:`ThreadingMock.DEFAULT_TIMEOUT`. + + .. method:: wait_until_called(*, timeout=UNSET) + + Waits until the mock is called. + + If a timeout was passed at the creation of the mock or if a timeout + argument is passed to this function, the function raises an + :exc:`AssertionError` if the call is not performed in time. + + >>> mock = ThreadingMock() + >>> thread = threading.Thread(target=mock) + >>> thread.start() + >>> mock.wait_until_called(timeout=1) + >>> thread.join() + + .. method:: wait_until_any_call(*args, **kwargs) + + Waits until the the mock is called with the specified arguments. + + If a timeout was passed at the creation of the mock + the function raises an :exc:`AssertionError` if the call is not performed in time. + + >>> mock = ThreadingMock() + >>> thread = threading.Thread(target=mock, args=("arg1", "arg2",), kwargs={"arg": "thing"}) + >>> thread.start() + >>> mock.wait_until_any_call("arg1", "arg2", arg="thing") + >>> thread.join() + + .. attribute:: DEFAULT_TIMEOUT + + Global default timeout in seconds to create instances of :class:`ThreadingMock`. + + .. versionadded:: 3.13 + + Calling ~~~~~~~ diff --git a/Lib/test/test_unittest/testmock/testthreadingmock.py b/Lib/test/test_unittest/testmock/testthreadingmock.py new file mode 100644 index 00000000000000..6219cc58abdc44 --- /dev/null +++ b/Lib/test/test_unittest/testmock/testthreadingmock.py @@ -0,0 +1,278 @@ +import time +import unittest +import concurrent.futures + +from unittest.mock import patch, ThreadingMock, call + + +class Something: + def method_1(self): + pass + + def method_2(self): + pass + + +class TestThreadingMock(unittest.TestCase): + def _call_after_delay(self, func, /, *args, **kwargs): + time.sleep(kwargs.pop("delay")) + func(*args, **kwargs) + + def setUp(self): + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=5) + + def tearDown(self): + self._executor.shutdown() + + def run_async(self, func, /, *args, delay=0, **kwargs): + self._executor.submit( + self._call_after_delay, func, *args, **kwargs, delay=delay + ) + + def _make_mock(self, *args, **kwargs): + return ThreadingMock(*args, **kwargs) + + def test_spec(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock) as m: + something = m() + + self.assertIsInstance(something.method_1, ThreadingMock) + self.assertIsInstance(something.method_1().method_2(), ThreadingMock) + + with self.assertRaises(AttributeError): + m.test + + def test_side_effect(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1.side_effect = [1] + + self.assertEqual(something.method_1(), 1) + + def test_instance_check(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + + self.assertIsInstance(something.method_1, ThreadingMock) + self.assertIsInstance(something.method_1().method_2(), ThreadingMock) + + def test_dynamic_child_mocks_are_threading_mocks(self): + waitable_mock = self._make_mock() + self.assertIsInstance(waitable_mock.child, ThreadingMock) + + def test_dynamic_child_mocks_inherit_timeout(self): + mock1 = self._make_mock() + self.assertIs(mock1._mock_wait_timeout, None) + mock2 = self._make_mock(timeout=2) + self.assertEqual(mock2._mock_wait_timeout, 2) + mock3 = self._make_mock(timeout=3) + self.assertEqual(mock3._mock_wait_timeout, 3) + + self.assertIs(mock1.child._mock_wait_timeout, None) + self.assertEqual(mock2.child._mock_wait_timeout, 2) + self.assertEqual(mock3.child._mock_wait_timeout, 3) + + self.assertEqual(mock2.really().__mul__().complex._mock_wait_timeout, 2) + + def test_no_name_clash(self): + waitable_mock = self._make_mock() + waitable_mock._event = "myevent" + waitable_mock.event = "myevent" + waitable_mock.timeout = "mytimeout" + waitable_mock("works") + waitable_mock.wait_until_called() + waitable_mock.wait_until_any_call("works") + + def test_wait_success(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, delay=0.01) + something.method_1.wait_until_called() + something.method_1.wait_until_any_call() + something.method_1.assert_called() + + def test_wait_success_with_instance_timeout(self): + waitable_mock = self._make_mock(timeout=1) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, delay=0.01) + something.method_1.wait_until_called() + something.method_1.wait_until_any_call() + something.method_1.assert_called() + + def test_wait_failed_with_instance_timeout(self): + waitable_mock = self._make_mock(timeout=0.01) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, delay=0.5) + self.assertRaises(AssertionError, waitable_mock.method_1.wait_until_called) + self.assertRaises( + AssertionError, waitable_mock.method_1.wait_until_any_call + ) + + def test_wait_success_with_timeout_override(self): + waitable_mock = self._make_mock(timeout=0.01) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, delay=0.05) + something.method_1.wait_until_called(timeout=1) + + def test_wait_failed_with_timeout_override(self): + waitable_mock = self._make_mock(timeout=1) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, delay=0.1) + with self.assertRaises(AssertionError): + something.method_1.wait_until_called(timeout=0.05) + with self.assertRaises(AssertionError): + something.method_1.wait_until_any_call(timeout=0.05) + + def test_wait_success_called_before(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1() + something.method_1.wait_until_called() + something.method_1.wait_until_any_call() + something.method_1.assert_called() + + def test_wait_magic_method(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1.__str__, delay=0.01) + something.method_1.__str__.wait_until_called() + something.method_1.__str__.assert_called() + + def test_wait_until_any_call_positional(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, 1, delay=0.1) + self.run_async(something.method_1, 2, delay=0.2) + self.run_async(something.method_1, 3, delay=0.3) + self.assertNotIn(call(1), something.method_1.mock_calls) + + something.method_1.wait_until_any_call(1) + something.method_1.assert_called_with(1) + self.assertNotIn(call(2), something.method_1.mock_calls) + self.assertNotIn(call(3), something.method_1.mock_calls) + + something.method_1.wait_until_any_call(3) + self.assertIn(call(2), something.method_1.mock_calls) + something.method_1.wait_until_any_call(2) + + def test_wait_until_any_call_keywords(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + self.run_async(something.method_1, a=1, delay=0.1) + self.run_async(something.method_1, b=2, delay=0.2) + self.run_async(something.method_1, c=3, delay=0.3) + self.assertNotIn(call(a=1), something.method_1.mock_calls) + + something.method_1.wait_until_any_call(a=1) + something.method_1.assert_called_with(a=1) + self.assertNotIn(call(b=2), something.method_1.mock_calls) + self.assertNotIn(call(c=3), something.method_1.mock_calls) + + something.method_1.wait_until_any_call(c=3) + self.assertIn(call(b=2), something.method_1.mock_calls) + something.method_1.wait_until_any_call(b=2) + + def test_wait_until_any_call_no_argument_fails_when_called_with_arg(self): + waitable_mock = self._make_mock(timeout=0.01) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1(1) + + something.method_1.assert_called_with(1) + with self.assertRaises(AssertionError): + something.method_1.wait_until_any_call() + + something.method_1() + something.method_1.wait_until_any_call() + + def test_wait_until_any_call_global_default(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = 0.01 + m = self._make_mock() + with self.assertRaises(AssertionError): + m.wait_until_any_call() + with self.assertRaises(AssertionError): + m.wait_until_called() + + m() + m.wait_until_any_call() + assert ThreadingMock.DEFAULT_TIMEOUT != 0.01 + + def test_wait_until_any_call_change_global_and_override(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = 0.01 + + m1 = self._make_mock() + self.run_async(m1, delay=0.1) + with self.assertRaises(AssertionError): + m1.wait_until_called() + + m2 = self._make_mock(timeout=1) + self.run_async(m2, delay=0.1) + m2.wait_until_called() + + m3 = self._make_mock() + self.run_async(m3, delay=0.1) + m3.wait_until_called(timeout=1) + + m4 = self._make_mock() + self.run_async(m4, delay=0.1) + m4.wait_until_called(timeout=None) + + m5 = self._make_mock(timeout=None) + self.run_async(m5, delay=0.1) + m5.wait_until_called() + + assert ThreadingMock.DEFAULT_TIMEOUT != 0.01 + + def test_reset_mock_resets_wait(self): + m = self._make_mock(timeout=0.01) + + with self.assertRaises(AssertionError): + m.wait_until_called() + with self.assertRaises(AssertionError): + m.wait_until_any_call() + m() + m.wait_until_called() + m.wait_until_any_call() + m.assert_called_once() + + m.reset_mock() + + with self.assertRaises(AssertionError): + m.wait_until_called() + with self.assertRaises(AssertionError): + m.wait_until_any_call() + m() + m.wait_until_called() + m.wait_until_any_call() + m.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index f10f9b04a5ab18..b542a9a2c51a9f 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -14,6 +14,7 @@ 'call', 'create_autospec', 'AsyncMock', + 'ThreadingMock', 'FILTER_DIR', 'NonCallableMock', 'NonCallableMagicMock', @@ -32,6 +33,7 @@ import builtins import pkgutil from asyncio import iscoroutinefunction +import threading from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -3003,6 +3005,98 @@ def __set__(self, obj, val): self(val) +_timeout_unset = sentinel.TIMEOUT_UNSET + +class ThreadingMixin(Base): + + DEFAULT_TIMEOUT = None + + def _get_child_mock(self, /, **kw): + if "timeout" in kw: + kw["timeout"] = kw.pop("timeout") + elif isinstance(kw.get("parent"), ThreadingMixin): + kw["timeout"] = kw["parent"]._mock_wait_timeout + elif isinstance(kw.get("_new_parent"), ThreadingMixin): + kw["timeout"] = kw["_new_parent"]._mock_wait_timeout + return super()._get_child_mock(**kw) + + def __init__(self, *args, timeout=_timeout_unset, **kwargs): + super().__init__(*args, **kwargs) + if timeout is _timeout_unset: + timeout = self.DEFAULT_TIMEOUT + self.__dict__["_mock_event"] = threading.Event() # Event for any call + self.__dict__["_mock_calls_events"] = [] # Events for each of the calls + self.__dict__["_mock_calls_events_lock"] = threading.Lock() + self.__dict__["_mock_wait_timeout"] = timeout + + def reset_mock(self, /, *args, **kwargs): + """ + See :func:`.Mock.reset_mock()` + """ + super().reset_mock(*args, **kwargs) + self.__dict__["_mock_event"] = threading.Event() + self.__dict__["_mock_calls_events"] = [] + + def __get_event(self, expected_args, expected_kwargs): + with self._mock_calls_events_lock: + for args, kwargs, event in self._mock_calls_events: + if (args, kwargs) == (expected_args, expected_kwargs): + return event + new_event = threading.Event() + self._mock_calls_events.append((expected_args, expected_kwargs, new_event)) + return new_event + + def _mock_call(self, *args, **kwargs): + ret_value = super()._mock_call(*args, **kwargs) + + call_event = self.__get_event(args, kwargs) + call_event.set() + + self._mock_event.set() + + return ret_value + + def wait_until_called(self, *, timeout=_timeout_unset): + """Wait until the mock object is called. + + `timeout` - time to wait for in seconds, waits forever otherwise. + Defaults to the constructor provided timeout. + Use None to block undefinetively. + """ + if timeout is _timeout_unset: + timeout = self._mock_wait_timeout + if not self._mock_event.wait(timeout=timeout): + msg = (f"{self._mock_name or 'mock'} was not called before" + f" timeout({timeout}).") + raise AssertionError(msg) + + def wait_until_any_call(self, *args, **kwargs): + """Wait until the mock object is called with given args. + + Waits for the timeout in seconds provided in the constructor. + """ + event = self.__get_event(args, kwargs) + if not event.wait(timeout=self._mock_wait_timeout): + expected_string = self._format_mock_call_signature(args, kwargs) + raise AssertionError(f'{expected_string} call not found') + + +class ThreadingMock(ThreadingMixin, MagicMixin, Mock): + """ + A mock that can be used to wait until on calls happening + in a different thread. + + The constructor can take a `timeout` argument which + controls the timeout in seconds for all `wait` calls of the mock. + + You can change the default timeout of all instances via the + `ThreadingMock.DEFAULT_TIMEOUT` attribute. + + If no timeout is set, it will block undefinetively. + """ + pass + + def seal(mock): """Disable the automatic generation of child mocks. diff --git a/Misc/NEWS.d/next/Library/2019-09-13-13-28-10.bpo-17013.NWcgE3.rst b/Misc/NEWS.d/next/Library/2019-09-13-13-28-10.bpo-17013.NWcgE3.rst new file mode 100644 index 00000000000000..ac746c45fa9ea8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-13-13-28-10.bpo-17013.NWcgE3.rst @@ -0,0 +1,3 @@ +Add ``ThreadingMock`` to :mod:`unittest.mock` that can be used to create +Mock objects that can wait until they are called. Patch by Karthikeyan +Singaravelan and Mario Corchero. From 5ccbbe5bb9a659fa8f2fe551428c84cc14015f44 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jul 2023 10:23:43 +0200 Subject: [PATCH 19/31] gh-106320: Move _PyUnicodeWriter to the internal C API (#106342) Move also _PyUnicode_FormatAdvancedWriter(). CJK codecs and multibytecodec.c now define the Py_BUILD_CORE_MODULE macro. --- Include/cpython/unicodeobject.h | 139 ----------------- Include/internal/pycore_complexobject.h | 2 + Include/internal/pycore_floatobject.h | 2 + Include/internal/pycore_unicodeobject.h | 145 +++++++++++++++++- Modules/cjkcodecs/cjkcodecs.h | 4 + Modules/cjkcodecs/multibytecodec.c | 4 + Modules/cjkcodecs/multibytecodec.h | 2 + Tools/c-analyzer/c_parser/preprocessor/gcc.py | 12 +- 8 files changed, 166 insertions(+), 144 deletions(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index dee8b27d3d97de..c5892a80d2c54d 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -480,131 +480,6 @@ PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar ( Py_ssize_t start, Py_ssize_t end); -/* --- _PyUnicodeWriter API ----------------------------------------------- */ - -typedef struct { - PyObject *buffer; - void *data; - int kind; - Py_UCS4 maxchar; - Py_ssize_t size; - Py_ssize_t pos; - - /* minimum number of allocated characters (default: 0) */ - Py_ssize_t min_length; - - /* minimum character (default: 127, ASCII) */ - Py_UCS4 min_char; - - /* If non-zero, overallocate the buffer (default: 0). */ - unsigned char overallocate; - - /* If readonly is 1, buffer is a shared string (cannot be modified) - and size is set to 0. */ - unsigned char readonly; -} _PyUnicodeWriter ; - -/* Initialize a Unicode writer. - * - * By default, the minimum buffer size is 0 character and overallocation is - * disabled. Set min_length, min_char and overallocate attributes to control - * the allocation of the buffer. */ -PyAPI_FUNC(void) -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); - -/* Prepare the buffer to write 'length' characters - with the specified maximum character. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ - (((MAXCHAR) <= (WRITER)->maxchar \ - && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ - ? 0 \ - : (((LENGTH) == 0) \ - ? 0 \ - : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) - -/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro - instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar); - -/* Prepare the buffer to have at least the kind KIND. - For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will - support characters in range U+000-U+FFFF. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ - ((KIND) <= (WRITER)->kind \ - ? 0 \ - : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) - -/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() - macro instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, - int kind); - -/* Append a Unicode character. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, - Py_UCS4 ch - ); - -/* Append a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, - PyObject *str /* Unicode string */ - ); - -/* Append a substring of a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, - PyObject *str, /* Unicode string */ - Py_ssize_t start, - Py_ssize_t end - ); - -/* Append an ASCII-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, - const char *str, /* ASCII-encoded byte string */ - Py_ssize_t len /* number of bytes, or -1 if unknown */ - ); - -/* Append a latin1-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, - const char *str, /* latin1-encoded byte string */ - Py_ssize_t len /* length in bytes */ - ); - -/* Get the value of the writer as a Unicode string. Clear the - buffer of the writer. Raise an exception and return NULL - on error. */ -PyAPI_FUNC(PyObject *) -_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) -_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); - - -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); - /* --- Manage the default encoding ---------------------------------------- */ /* Returns a pointer to the default encoding (UTF-8) of the @@ -774,20 +649,6 @@ PyAPI_FUNC(PyObject *) _PyUnicode_XStrip( PyObject *sepobj ); -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar); - /* === Characters Type APIs =============================================== */ /* These should not be used directly. Use the Py_UNICODE_IS* and diff --git a/Include/internal/pycore_complexobject.h b/Include/internal/pycore_complexobject.h index fb344b7bb79bd8..7843c0a008ebb6 100644 --- a/Include/internal/pycore_complexobject.h +++ b/Include/internal/pycore_complexobject.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_unicodeobject.h" // _PyUnicodeWriter + /* Operations on complex numbers from complexmodule.c */ PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 27c63bc87f3ee3..6abba04033d281 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -9,6 +9,8 @@ extern "C" { #endif +#include "pycore_unicodeobject.h" // _PyUnicodeWriter + /* runtime lifecycle */ extern void _PyFloat_InitState(PyInterpreterState *); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 1bb0f366e78163..a8c7f1957f3600 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -14,7 +14,148 @@ extern "C" { void _PyUnicode_ExactDealloc(PyObject *op); Py_ssize_t _PyUnicode_InternedSize(void); -/* runtime lifecycle */ +/* --- _PyUnicodeWriter API ----------------------------------------------- */ + +typedef struct { + PyObject *buffer; + void *data; + int kind; + Py_UCS4 maxchar; + Py_ssize_t size; + Py_ssize_t pos; + + /* minimum number of allocated characters (default: 0) */ + Py_ssize_t min_length; + + /* minimum character (default: 127, ASCII) */ + Py_UCS4 min_char; + + /* If non-zero, overallocate the buffer (default: 0). */ + unsigned char overallocate; + + /* If readonly is 1, buffer is a shared string (cannot be modified) + and size is set to 0. */ + unsigned char readonly; +} _PyUnicodeWriter ; + +/* Initialize a Unicode writer. + * + * By default, the minimum buffer size is 0 character and overallocation is + * disabled. Set min_length, min_char and overallocate attributes to control + * the allocation of the buffer. */ +PyAPI_FUNC(void) +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); + +/* Prepare the buffer to write 'length' characters + with the specified maximum character. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ + (((MAXCHAR) <= (WRITER)->maxchar \ + && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ + ? 0 \ + : (((LENGTH) == 0) \ + ? 0 \ + : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) + +/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro + instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar); + +/* Prepare the buffer to have at least the kind KIND. + For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will + support characters in range U+000-U+FFFF. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ + ((KIND) <= (WRITER)->kind \ + ? 0 \ + : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) + +/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() + macro instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + int kind); + +/* Append a Unicode character. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, + Py_UCS4 ch + ); + +/* Append a Unicode string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, + PyObject *str /* Unicode string */ + ); + +/* Append a substring of a Unicode string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, + PyObject *str, /* Unicode string */ + Py_ssize_t start, + Py_ssize_t end + ); + +/* Append an ASCII-encoded byte string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, + const char *str, /* ASCII-encoded byte string */ + Py_ssize_t len /* number of bytes, or -1 if unknown */ + ); + +/* Append a latin1-encoded byte string. + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, + const char *str, /* latin1-encoded byte string */ + Py_ssize_t len /* length in bytes */ + ); + +/* Get the value of the writer as a Unicode string. Clear the + buffer of the writer. Raise an exception and return NULL + on error. */ +PyAPI_FUNC(PyObject *) +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); + + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +/* --- Methods & Slots ---------------------------------------------------- */ + +/* Using explicit passed-in values, insert the thousands grouping + into the string pointed to by buffer. For the argument descriptions, + see Objects/stringlib/localeutil.h */ +PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( + _PyUnicodeWriter *writer, + Py_ssize_t n_buffer, + PyObject *digits, + Py_ssize_t d_pos, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + PyObject *thousands_sep, + Py_UCS4 *maxchar); + +/* --- Runtime lifecycle -------------------------------------------------- */ extern void _PyUnicode_InitState(PyInterpreterState *); extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *); @@ -24,7 +165,7 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *); extern PyTypeObject _PyUnicodeASCIIIter_Type; -/* other API */ +/* --- Other API ---------------------------------------------------------- */ struct _Py_unicode_runtime_ids { PyThread_type_lock lock; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 48cdcfb3ad3087..97290aac3ba439 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -7,6 +7,10 @@ #ifndef _CJKCODECS_H_ #define _CJKCODECS_H_ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "multibytecodec.h" diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index cf437d09f1fe8d..3febd1a832f9cc 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -4,6 +4,10 @@ * Written by Hye-Shik Chang */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "structmember.h" // PyMemberDef #include "multibytecodec.h" diff --git a/Modules/cjkcodecs/multibytecodec.h b/Modules/cjkcodecs/multibytecodec.h index f59362205d26fc..5b85e2e3de316d 100644 --- a/Modules/cjkcodecs/multibytecodec.h +++ b/Modules/cjkcodecs/multibytecodec.h @@ -10,6 +10,8 @@ extern "C" { #endif +#include "pycore_unicodeobject.h" // _PyUnicodeWriter + #ifdef uint16_t typedef uint16_t ucs2_t, DBCHAR; #else diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 0929f7111eac92..415a2ba63dfe68 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -3,6 +3,14 @@ from . import common as _common +# Modules/socketmodule.h uses pycore_time.h which needs the Py_BUILD_CORE +# macro. Usually it's defined by the C file which includes it. +# Other header files have a similar issue. +NEED_BUILD_CORE = { + 'cjkcodecs.h', + 'multibytecodec.h', + 'socketmodule.h', +} TOOL = 'gcc' @@ -62,9 +70,7 @@ def preprocess(filename, filename = _normpath(filename, cwd) postargs = POST_ARGS - if os.path.basename(filename) == 'socketmodule.h': - # Modules/socketmodule.h uses pycore_time.h which needs Py_BUILD_CORE. - # Usually it's defined by the C file which includes it. + if os.path.basename(filename) in NEED_BUILD_CORE: postargs += ('-DPy_BUILD_CORE=1',) text = _common.preprocess( From 35963da40fb7bc93c55d94caf58ff9e268df951d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jul 2023 11:39:11 +0200 Subject: [PATCH 20/31] gh-106320: Create pycore_modsupport.h header file (#106355) Remove the following functions from the C API, move them to the internal C API: add a new pycore_modsupport.h internal header file: * PyModule_CreateInitialized() * _PyArg_NoKwnames() * _Py_VaBuildStack() No longer export these functions. --- Include/cpython/modsupport.h | 12 ------------ Include/internal/pycore_modsupport.h | 29 ++++++++++++++++++++++++++++ Makefile.pre.in | 1 + Modules/_operator.c | 4 +++- Objects/boolobject.c | 5 +++-- Objects/call.c | 1 + Objects/enumobject.c | 1 + Objects/floatobject.c | 1 + Objects/listobject.c | 1 + Objects/moduleobject.c | 3 ++- Objects/rangeobject.c | 3 ++- Objects/setobject.c | 1 + Objects/tupleobject.c | 1 + Objects/typeobject.c | 1 + Objects/weakrefobject.c | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ Python/bltinmodule.c | 3 ++- Python/instrumentation.c | 1 + Python/sysmodule.c | 1 + 20 files changed, 56 insertions(+), 18 deletions(-) create mode 100644 Include/internal/pycore_modsupport.h diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index a5d95d15440df1..376336b13dcf8a 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -11,12 +11,9 @@ PyAPI_FUNC(int) _PyArg_UnpackStack( ...); PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); -PyAPI_FUNC(int) _PyArg_NoKwnames(const char *funcname, PyObject *kwnames); PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); #define _PyArg_NoKeywords(funcname, kwargs) \ ((kwargs) == NULL || _PyArg_NoKeywords((funcname), (kwargs))) -#define _PyArg_NoKwnames(funcname, kwnames) \ - ((kwnames) == NULL || _PyArg_NoKwnames((funcname), (kwnames))) #define _PyArg_NoPositional(funcname, args) \ ((args) == NULL || _PyArg_NoPositional((funcname), (args))) @@ -29,13 +26,6 @@ PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) -PyAPI_FUNC(PyObject **) _Py_VaBuildStack( - PyObject **small_stack, - Py_ssize_t small_stack_len, - const char *format, - va_list va, - Py_ssize_t *p_nargs); - typedef struct _PyArg_Parser { int initialized; const char *format; @@ -83,5 +73,3 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ (minpos), (maxpos), (minkw), (buf))) - -PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(PyModuleDef*, int apiver); diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h new file mode 100644 index 00000000000000..e577c6ba856b77 --- /dev/null +++ b/Include/internal/pycore_modsupport.h @@ -0,0 +1,29 @@ +#ifndef Py_INTERNAL_MODSUPPORT_H +#define Py_INTERNAL_MODSUPPORT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +extern int _PyArg_NoKwnames(const char *funcname, PyObject *kwnames); +#define _PyArg_NoKwnames(funcname, kwnames) \ + ((kwnames) == NULL || _PyArg_NoKwnames((funcname), (kwnames))) + +extern PyObject ** _Py_VaBuildStack( + PyObject **small_stack, + Py_ssize_t small_stack_len, + const char *format, + va_list va, + Py_ssize_t *p_nargs); + +extern PyObject* _PyModule_CreateInitialized(PyModuleDef*, int apiver); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MODSUPPORT_H + diff --git a/Makefile.pre.in b/Makefile.pre.in index 7560d17e3796f0..dcb3b5eb06a1e9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1761,6 +1761,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_list.h \ $(srcdir)/Include/internal/pycore_long.h \ + $(srcdir)/Include/internal/pycore_modsupport.h \ $(srcdir)/Include/internal/pycore_moduleobject.h \ $(srcdir)/Include/internal/pycore_namespace.h \ $(srcdir)/Include/internal/pycore_object.h \ diff --git a/Modules/_operator.c b/Modules/_operator.c index 153e9e9e2f92c4..108f45fb6dad93 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1,7 +1,9 @@ #include "Python.h" +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "structmember.h" // PyMemberDef #include "pycore_runtime.h" // _Py_ID() + +#include "structmember.h" // PyMemberDef #include "clinic/_operator.c.h" typedef struct { diff --git a/Objects/boolobject.c b/Objects/boolobject.c index f43e26f3f24e77..bbb187cb7121e7 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -1,8 +1,9 @@ /* Boolean type, a subtype of int */ #include "Python.h" -#include "pycore_object.h" // _Py_FatalRefcountError() -#include "pycore_long.h" // FALSE_TAG TRUE_TAG +#include "pycore_long.h" // FALSE_TAG TRUE_TAG +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _Py_FatalRefcountError() #include "pycore_runtime.h" // _Py_ID() #include diff --git a/Objects/call.c b/Objects/call.c index 16c41ffe1d09b5..5045c0dc92843b 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -2,6 +2,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgsTstate() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_dict.h" // _PyDict_FromItems() +#include "pycore_modsupport.h" // _Py_VaBuildStack() #include "pycore_object.h" // _PyCFunctionWithKeywords_TrampolineCall() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Objects/enumobject.c b/Objects/enumobject.c index c9d90584c26b7d..556666779d8522 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetOne() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "clinic/enumobject.c.h" diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 83a263c0d9c67e..fa55481f09dec0 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -9,6 +9,7 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _PyInterpreterState.float_state #include "pycore_long.h" // _PyLong_GetOne() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Objects/listobject.c b/Objects/listobject.c index f1f324f7439b43..98fa08962b6aad 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -5,6 +5,7 @@ #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_state, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_tuple.h" // _PyTuple_FromArray() #include diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index bda25c881845ce..d4fccae65244c5 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -5,8 +5,9 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_object.h" // _PyType_AllocNoTrack -#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_modsupport.h" // _PyModule_CreateInitialized() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "structmember.h" // PyMemberDef diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index beb86b9623bdbc..6dc41d71287cab 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -2,8 +2,9 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_range.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_range.h" #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "structmember.h" // PyMemberDef diff --git a/Objects/setobject.c b/Objects/setobject.c index 58f0ae73c0c403..4ac541b9509752 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -32,6 +32,7 @@ */ #include "Python.h" +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include // offsetof() diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 991edcc86677de..e85af2b75e4738 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -5,6 +5,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() /*[clinic input] diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3d3a63a75bd2fb..87519efef081c3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8,6 +8,7 @@ #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_Occurred() diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index f3f6c86637e9de..bac3e79bb7c250 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "structmember.h" // PyMemberDef diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 79ce2d3d14017e..760962e4c4b6a9 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -241,6 +241,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index d47a22909e1e3a..aaebe1908e30da 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -627,6 +627,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 68fe315338a54d..9fe0067daa678c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -4,13 +4,14 @@ #include #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_long.h" // _PyLong_CompactValue +#include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() #include "pycore_pyerrors.h" // _PyErr_NoMemory() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_ceval.h" // _PyEval_Vector() #include "clinic/bltinmodule.c.h" diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 03d7d2f215af7c..e29748f0ad9872 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -3,6 +3,7 @@ #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_long.h" +#include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode.h" diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 56d771f70ef538..0ac6edc5a16f88 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -20,6 +20,7 @@ Data members: #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD +#include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() From 18fedd04a7b4d54b5eb7a34a1e1c9ca42219f882 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jul 2023 12:06:54 +0200 Subject: [PATCH 21/31] gh-104050: Annotate Argument Clinic DSLParser attributes (#106357) --- Tools/clinic/clinic.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 27d3d572868883..125604bff97f7c 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4293,6 +4293,19 @@ def dedent(self, line): StateKeeper = Callable[[str | None], None] class DSLParser: + function: Function | None + state: StateKeeper + keyword_only: bool + positional_only: bool + group: int + parameter_state: int + seen_positional_with_default: bool + indent: IndentStack + kind: str + coexist: bool + parameter_continuation: str + preserve_output: bool + def __init__(self, clinic: Clinic) -> None: self.clinic = clinic @@ -4312,7 +4325,7 @@ def __init__(self, clinic: Clinic) -> None: def reset(self) -> None: self.function = None - self.state: StateKeeper = self.state_dsl_start + self.state = self.state_dsl_start self.parameter_indent = None self.keyword_only = False self.positional_only = False From c5afc97fc2f149758f597c21f65b4a1c770bb827 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jul 2023 12:48:50 +0200 Subject: [PATCH 22/31] gh-106320: Remove private _PyErr C API functions (#106356) Remove private _PyErr C API functions: move them to the internal C API (pycore_pyerrors.h). --- Include/cpython/pyerrors.h | 42 -------------------------- Include/internal/pycore_pyerrors.h | 48 ++++++++++++++++++++++++++++++ Modules/_io/bufferedio.c | 1 + Modules/_sqlite/cursor.c | 6 ++++ Objects/moduleobject.c | 5 ++-- Objects/obmalloc.c | 4 +-- Objects/unicodeobject.c | 1 + Parser/pegen_errors.c | 1 + Python/importdl.c | 1 + 9 files changed, 63 insertions(+), 46 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 156665cbdb1ba4..5c128211bd525a 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -91,31 +91,14 @@ typedef PyOSErrorObject PyWindowsErrorObject; /* Error handling definitions */ PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); -PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); -PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); -PyAPI_FUNC(void) _PyErr_SetHandledException(PyThreadState *, PyObject *); -PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); /* Context manipulation (PEP 3134) */ Py_DEPRECATED(3.12) PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); -/* Like PyErr_Format(), but saves current exception as __context__ and - __cause__. - */ -PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause( - PyObject *exception, - const char *format, /* ASCII-encoded string */ - ... - ); - /* In exceptions.c */ -PyAPI_FUNC(int) _PyException_AddNote( - PyObject *exc, - PyObject *note); - PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( PyObject *orig, PyObject *excs); @@ -123,7 +106,6 @@ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( /* In signalmodule.c */ int PySignal_SetWakeupFd(int fd); -PyAPI_FUNC(int) _PyErr_CheckSignals(void); /* Support for adding program text to SyntaxErrors */ @@ -143,18 +125,6 @@ PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); -PyAPI_FUNC(PyObject *) _PyErr_ProgramDecodedTextObject( - PyObject *filename, - int lineno, - const char* encoding); - -PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( - PyObject *object, - Py_ssize_t start, - Py_ssize_t end, - const char *reason /* UTF-8 encoded string */ - ); - PyAPI_FUNC(void) _PyErr_WriteUnraisableMsg( const char *err_msg, PyObject *obj); @@ -163,16 +133,4 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( const char *func, const char *message); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( - const char *func, - const char *format, - ...); - -extern PyObject *_PyErr_SetImportErrorWithNameFrom( - PyObject *, - PyObject *, - PyObject *, - PyObject *); - - #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message)) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index d75bef0c3b5d8f..e3ba4b75e3cfc3 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -9,6 +9,54 @@ extern "C" { #endif +/* Error handling definitions */ + +PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); +PyAPI_FUNC(void) _PyErr_SetHandledException(PyThreadState *, PyObject *); +PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); + +/* Like PyErr_Format(), but saves current exception as __context__ and + __cause__. + */ +PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause( + PyObject *exception, + const char *format, /* ASCII-encoded string */ + ... + ); + +PyAPI_FUNC(int) _PyException_AddNote( + PyObject *exc, + PyObject *note); + +PyAPI_FUNC(int) _PyErr_CheckSignals(void); + +/* Support for adding program text to SyntaxErrors */ + +PyAPI_FUNC(PyObject *) _PyErr_ProgramDecodedTextObject( + PyObject *filename, + int lineno, + const char* encoding); + +PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( + PyObject *object, + Py_ssize_t start, + Py_ssize_t end, + const char *reason /* UTF-8 encoded string */ + ); + +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( + const char *func, + const char *format, + ...); + +extern PyObject *_PyErr_SetImportErrorWithNameFrom( + PyObject *, + PyObject *, + PyObject *, + PyObject *); + + /* runtime lifecycle */ extern PyStatus _PyErr_InitTypes(PyInterpreterState *); diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 9c6a9cddba2258..4d120d4e8af3a7 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "structmember.h" // PyMemberDef #include "_iomodule.h" diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index caeedbddb8d88b..dba8ab61e41e70 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -21,11 +21,17 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "cursor.h" #include "microprotocols.h" #include "module.h" #include "util.h" +#include "pycore_pyerrors.h" // _PyErr_FormatFromCause() + typedef enum { TYPE_LONG, TYPE_FLOAT, diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d4fccae65244c5..3e500b555e70ff 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -4,9 +4,10 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.importlib -#include "pycore_object.h" // _PyType_AllocNoTrack -#include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() +#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_object.h" // _PyType_AllocNoTrack +#include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "structmember.h" // PyMemberDef diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 9620a8fbb44cac..eb68d7c030d293 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,10 +2,10 @@ #include "Python.h" #include "pycore_code.h" // stats -#include "pycore_pystate.h" // _PyInterpreterState_GET - #include "pycore_obmalloc.h" +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pymem.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET #include // malloc() #include diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 79402714f23b9d..12e379d0c7e6cd 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -50,6 +50,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_long.h" // _PyLong_FormatWriter() #include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() #include "pycore_pathconfig.h" // _Py_DumpPathConfig() +#include "pycore_pyerrors.h" // _PyUnicodeTranslateError_Create() #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index af529057f50e70..e543d40ccd8ab7 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -1,6 +1,7 @@ #include #include +#include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject() #include "tokenizer.h" #include "pegen.h" diff --git a/Python/importdl.c b/Python/importdl.c index eb6b808ecba1d5..9ab0a5ad33aaac 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_call.h" #include "pycore_import.h" +#include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" #include "pycore_runtime.h" From 60e41a0cbbe1a43e504b50a454efe653aba78a1a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jul 2023 13:00:35 +0200 Subject: [PATCH 23/31] gh-104146: Remove unused attr 'parameter_indent' from clinic.DLParser (#106358) --- Tools/clinic/clinic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 125604bff97f7c..2e46131d4e336a 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4326,7 +4326,6 @@ def __init__(self, clinic: Clinic) -> None: def reset(self) -> None: self.function = None self.state = self.state_dsl_start - self.parameter_indent = None self.keyword_only = False self.positional_only = False self.group = 0 From 0da4c883cf4185efe27b711c3e0a1e6e94397610 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jul 2023 15:14:59 +0200 Subject: [PATCH 24/31] gh-106359: Fix corner case bugs in Argument Clinic converter parser (#106361) DSLParser.parse_converter() could return unusable kwdicts in some rare cases Co-authored-by: Alex Waygood --- Lib/test/test_clinic.py | 16 ++++++++++++++++ ...023-07-03-14-06-19.gh-issue-106359.RfJuR0.rst | 2 ++ Tools/clinic/clinic.py | 16 +++++++++------- 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-07-03-14-06-19.gh-issue-106359.RfJuR0.rst diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a6ce9c2570cb62..fab2d2bb0bdbe4 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -813,6 +813,22 @@ def test_other_bizarre_things_in_annotations_fail(self): ) self.assertEqual(s, expected_failure_message) + def test_kwarg_splats_disallowed_in_function_call_annotations(self): + expected_error_msg = ( + "Error on line 0:\n" + "Cannot use a kwarg splat in a function-call annotation\n" + ) + dataset = ( + 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{"bang": 42})', + 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', + ) + for fn in dataset: + with self.subTest(fn=fn): + out = self.parse_function_should_fail(fn) + self.assertEqual(out, expected_error_msg) + def test_unused_param(self): block = self.parse(""" module foo diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-07-03-14-06-19.gh-issue-106359.RfJuR0.rst b/Misc/NEWS.d/next/Tools-Demos/2023-07-03-14-06-19.gh-issue-106359.RfJuR0.rst new file mode 100644 index 00000000000000..600c265391ec5b --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-07-03-14-06-19.gh-issue-106359.RfJuR0.rst @@ -0,0 +1,2 @@ +Argument Clinic now explicitly forbids "kwarg splats" in function calls used as +annotations. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2e46131d4e336a..47038f98aabd75 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4291,6 +4291,7 @@ def dedent(self, line): StateKeeper = Callable[[str | None], None] +ConverterArgs = dict[str, Any] class DSLParser: function: Function | None @@ -5033,10 +5034,10 @@ def bad_node(self, node): key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p - KwargDict = dict[str | None, Any] - @staticmethod - def parse_converter(annotation: ast.expr | None) -> tuple[str, bool, KwargDict]: + def parse_converter( + annotation: ast.expr | None + ) -> tuple[str, bool, ConverterArgs]: match annotation: case ast.Constant(value=str() as value): return value, True, {} @@ -5044,10 +5045,11 @@ def parse_converter(annotation: ast.expr | None) -> tuple[str, bool, KwargDict]: return name, False, {} case ast.Call(func=ast.Name(name)): symbols = globals() - kwargs = { - node.arg: eval_ast_expr(node.value, symbols) - for node in annotation.keywords - } + kwargs: ConverterArgs = {} + for node in annotation.keywords: + if not isinstance(node.arg, str): + fail("Cannot use a kwarg splat in a function-call annotation") + kwargs[node.arg] = eval_ast_expr(node.value, symbols) return name, False, kwargs case _: fail( From 2e2daac2f11cbd762601382e759048fdbabba5bc Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jul 2023 16:03:31 +0200 Subject: [PATCH 25/31] gh-104050: Add more type hints to Argument Clinic DSLParser() (#106354) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 47038f98aabd75..41d4274fc744e0 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4,6 +4,7 @@ # Copyright 2012-2013 by Larry Hastings. # Licensed to the PSF under a contributor agreement. # +from __future__ import annotations import abc import ast @@ -2448,7 +2449,7 @@ def __init__( cls: Class | None = None, c_basename: str | None = None, full_name: str | None = None, - return_converter: ReturnConverterType, + return_converter: CReturnConverter, return_annotation = inspect.Signature.empty, docstring: str | None = None, kind: str = CALLABLE, @@ -2467,7 +2468,7 @@ def __init__( self.docstring = docstring or '' self.kind = kind self.coexist = coexist - self.self_converter = None + self.self_converter: self_converter | None = None # docstring_only means "don't generate a machine-readable # signature, just a normal docstring". it's True for # functions with optional groups because we can't represent @@ -2531,7 +2532,7 @@ class Parameter: def __init__( self, name: str, - kind: str, + kind: inspect._ParameterKind, *, default = inspect.Parameter.empty, function: Function, @@ -4539,7 +4540,7 @@ def state_dsl_start(self, line: str | None) -> None: self.next(self.state_modulename_name, line) - def state_modulename_name(self, line): + def state_modulename_name(self, line: str | None) -> None: # looking for declaration, which establishes the leftmost column # line should be # modulename.fnname [as c_basename] [-> return annotation] @@ -4556,13 +4557,14 @@ def state_modulename_name(self, line): # this line is permitted to start with whitespace. # we'll call this number of spaces F (for "function"). - if not line.strip(): + if not self.valid_line(line): return self.indent.infer(line) # are we cloning? before, equals, existing = line.rpartition('=') + c_basename: str | None if equals: full_name, _, c_basename = before.partition(' as ') full_name = full_name.strip() @@ -4665,8 +4667,9 @@ def state_modulename_name(self, line): if cls and type == "PyObject *": kwargs['type'] = cls.typedef sc = self.function.self_converter = self_converter(name, name, self.function, **kwargs) - p_self = Parameter(sc.name, inspect.Parameter.POSITIONAL_ONLY, function=self.function, converter=sc) - self.function.parameters[sc.name] = p_self + p_self = Parameter(name, inspect.Parameter.POSITIONAL_ONLY, + function=self.function, converter=sc) + self.function.parameters[name] = p_self (cls or module).functions.append(self.function) self.next(self.state_parameters_start) @@ -4740,7 +4743,7 @@ def state_modulename_name(self, line): ps_start, ps_left_square_before, ps_group_before, ps_required, \ ps_optional, ps_group_after, ps_right_square_after = range(7) - def state_parameters_start(self, line: str) -> None: + def state_parameters_start(self, line: str | None) -> None: if not self.valid_line(line): return From 58906213cc5d8f2be311664766b4923ef29dae1f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 3 Jul 2023 08:25:22 -0600 Subject: [PATCH 26/31] gh-91053: make func watcher tests resilient to other func watchers (#106286) --- Modules/_testcapi/watchers.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index d2c71fb401d36a..7167943fffab39 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -432,9 +432,9 @@ allocate_too_many_code_watchers(PyObject *self, PyObject *args) // Test function watchers -#define NUM_FUNC_WATCHERS 2 -static PyObject *pyfunc_watchers[NUM_FUNC_WATCHERS]; -static int func_watcher_ids[NUM_FUNC_WATCHERS] = {-1, -1}; +#define NUM_TEST_FUNC_WATCHERS 2 +static PyObject *pyfunc_watchers[NUM_TEST_FUNC_WATCHERS]; +static int func_watcher_ids[NUM_TEST_FUNC_WATCHERS] = {-1, -1}; static PyObject * get_id(PyObject *obj) @@ -508,7 +508,7 @@ second_func_watcher_callback(PyFunction_WatchEvent event, return call_pyfunc_watcher(pyfunc_watchers[1], event, func, new_value); } -static PyFunction_WatchCallback func_watcher_callbacks[NUM_FUNC_WATCHERS] = { +static PyFunction_WatchCallback func_watcher_callbacks[NUM_TEST_FUNC_WATCHERS] = { first_func_watcher_callback, second_func_watcher_callback }; @@ -533,26 +533,25 @@ add_func_watcher(PyObject *self, PyObject *func) return NULL; } int idx = -1; - for (int i = 0; i < NUM_FUNC_WATCHERS; i++) { + for (int i = 0; i < NUM_TEST_FUNC_WATCHERS; i++) { if (func_watcher_ids[i] == -1) { idx = i; break; } } if (idx == -1) { - PyErr_SetString(PyExc_RuntimeError, "no free watchers"); - return NULL; - } - PyObject *result = PyLong_FromLong(idx); - if (result == NULL) { + PyErr_SetString(PyExc_RuntimeError, "no free test watchers"); return NULL; } func_watcher_ids[idx] = PyFunction_AddWatcher(func_watcher_callbacks[idx]); if (func_watcher_ids[idx] < 0) { - Py_DECREF(result); return NULL; } pyfunc_watchers[idx] = Py_NewRef(func); + PyObject *result = PyLong_FromLong(func_watcher_ids[idx]); + if (result == NULL) { + return NULL; + } return result; } @@ -569,7 +568,7 @@ clear_func_watcher(PyObject *self, PyObject *watcher_id_obj) return NULL; } int idx = -1; - for (int i = 0; i < NUM_FUNC_WATCHERS; i++) { + for (int i = 0; i < NUM_TEST_FUNC_WATCHERS; i++) { if (func_watcher_ids[i] == wid) { idx = i; break; From 2028a4f6d996d2a46cbc33d0b65fdae284ee71fc Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 3 Jul 2023 13:05:11 -0700 Subject: [PATCH 27/31] gh-106290: Fix edge cases around uops (#106319) - Tweak uops debugging output - Fix the bug from gh-106290 - Rename `SET_IP` to `SAVE_IP` (per https://github.com/faster-cpython/ideas/issues/558) - Add a `SAVE_IP` uop at the start of the trace (ditto) - Allow `unbound_local_error`; this gives us uops for `LOAD_FAST_CHECK`, `LOAD_CLOSURE`, and `DELETE_FAST` - Longer traces - Support `STORE_FAST_LOAD_FAST`, `STORE_FAST_STORE_FAST` - Add deps on pycore_uops.h to Makefile(.pre.in) --- Include/internal/pycore_uops.h | 2 +- Makefile.pre.in | 1 + Python/ceval.c | 62 ++--- Python/executor_cases.c.h | 291 +++++++++++++----------- Python/opcode_metadata.h | 6 +- Python/optimizer.c | 132 ++++++----- Tools/cases_generator/generate_cases.py | 14 +- 7 files changed, 274 insertions(+), 234 deletions(-) diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index 0e88d7e7f4a3bd..5ed275fb857679 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#define _Py_UOP_MAX_TRACE_LENGTH 16 +#define _Py_UOP_MAX_TRACE_LENGTH 32 typedef struct { int opcode; diff --git a/Makefile.pre.in b/Makefile.pre.in index dcb3b5eb06a1e9..41623bd2f1da7f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1800,6 +1800,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_unionobject.h \ $(srcdir)/Include/internal/pycore_unicodeobject.h \ $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ + $(srcdir)/Include/internal/pycore_uops.h \ $(srcdir)/Include/internal/pycore_warnings.h \ $(srcdir)/Include/internal/pycore_weakref.h \ $(DTRACE_HEADERS) \ diff --git a/Python/ceval.c b/Python/ceval.c index 80ae85c24f0540..6ce1a6a636ed71 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2773,24 +2773,26 @@ void Py_LeaveRecursiveCall(void) _PyInterpreterFrame * _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer) { -#ifdef LLTRACE +#ifdef Py_DEBUG char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); int lltrace = 0; if (uop_debug != NULL && *uop_debug >= '0') { lltrace = *uop_debug - '0'; // TODO: Parse an int and all that } - if (lltrace >= 2) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *instr = frame->prev_instr + 1; - fprintf(stderr, - "Entering _PyUopExecute for %s (%s:%d) at offset %ld\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); - } +#define DPRINTF(level, ...) \ + if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } +#else +#define DPRINTF(level, ...) #endif + DPRINTF(3, + "Entering _PyUopExecute for %s (%s:%d) at offset %ld\n", + PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), + PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), + _PyFrame_GetCode(frame)->co_firstlineno, + (long)(frame->prev_instr + 1 - + (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); + PyThreadState *tstate = _PyThreadState_GET(); _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; @@ -2803,7 +2805,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject } OBJECT_STAT_INC(optimization_traces_executed); - _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive - 1; + _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; int pc = 0; int opcode; uint64_t operand; @@ -2812,14 +2814,11 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject opcode = self->trace[pc].opcode; operand = self->trace[pc].operand; oparg = (int)operand; -#ifdef LLTRACE - if (lltrace >= 3) { - const char *opname = opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]; - int stack_level = (int)(stack_pointer - _PyFrame_Stackbase(frame)); - fprintf(stderr, " uop %s, operand %" PRIu64 ", stack_level %d\n", - opname, operand, stack_level); - } -#endif + DPRINTF(3, + " uop %s, operand %" PRIu64 ", stack_level %d\n", + opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], + operand, + (int)(stack_pointer - _PyFrame_Stackbase(frame))); pc++; OBJECT_STAT_INC(optimization_uops_executed); switch (opcode) { @@ -2828,7 +2827,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject #define ENABLE_SPECIALIZATION 0 #include "executor_cases.c.h" - case SET_IP: + case SAVE_IP: { frame->prev_instr = ip_offset + oparg; break; @@ -2836,6 +2835,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject case EXIT_TRACE: { + frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return frame; @@ -2850,6 +2850,13 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject } } +unbound_local_error: + format_exc_check_arg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + goto error; + pop_4_error: STACK_SHRINK(1); pop_3_error: @@ -2861,11 +2868,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject error: // On ERROR_IF we return NULL as the frame. // The caller recovers the frame from cframe.current_frame. -#ifdef LLTRACE - if (lltrace >= 2) { - fprintf(stderr, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - } -#endif + DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return NULL; @@ -2873,11 +2876,8 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject deoptimize: // On DEOPT_IF we just repeat the last instruction. // This presumes nothing was popped from the stack (nor pushed). -#ifdef LLTRACE - if (lltrace >= 2) { - fprintf(stderr, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); - } -#endif + DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); + frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return frame; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 546b3d9f50ac76..d1e0443db613d2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7,13 +7,25 @@ break; } + case LOAD_FAST_CHECK: { + PyObject *value; + #line 182 "Python/bytecodes.c" + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); + #line 17 "Python/executor_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = value; + break; + } + case LOAD_FAST: { PyObject *value; #line 188 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 17 "Python/executor_cases.c.h" + #line 29 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; break; @@ -25,7 +37,7 @@ value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; - #line 29 "Python/executor_cases.c.h" + #line 41 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; break; @@ -36,7 +48,7 @@ #line 209 "Python/bytecodes.c" value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - #line 40 "Python/executor_cases.c.h" + #line 52 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; break; @@ -46,7 +58,7 @@ PyObject *value = stack_pointer[-1]; #line 214 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 50 "Python/executor_cases.c.h" + #line 62 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -54,7 +66,7 @@ case POP_TOP: { PyObject *value = stack_pointer[-1]; #line 237 "Python/bytecodes.c" - #line 58 "Python/executor_cases.c.h" + #line 70 "Python/executor_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); break; @@ -64,7 +76,7 @@ PyObject *res; #line 241 "Python/bytecodes.c" res = NULL; - #line 68 "Python/executor_cases.c.h" + #line 80 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -75,7 +87,7 @@ PyObject *receiver = stack_pointer[-2]; #line 260 "Python/bytecodes.c" Py_DECREF(receiver); - #line 79 "Python/executor_cases.c.h" + #line 91 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; break; @@ -86,11 +98,11 @@ PyObject *res; #line 275 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 90 "Python/executor_cases.c.h" + #line 102 "Python/executor_cases.c.h" Py_DECREF(value); #line 277 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 94 "Python/executor_cases.c.h" + #line 106 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -101,7 +113,7 @@ #line 281 "Python/bytecodes.c" assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; - #line 105 "Python/executor_cases.c.h" + #line 117 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -111,7 +123,7 @@ #line 313 "Python/bytecodes.c" DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); - #line 115 "Python/executor_cases.c.h" + #line 127 "Python/executor_cases.c.h" break; } @@ -126,12 +138,12 @@ res = Py_False; } else { - #line 130 "Python/executor_cases.c.h" + #line 142 "Python/executor_cases.c.h" Py_DECREF(value); #line 326 "Python/bytecodes.c" res = Py_True; } - #line 135 "Python/executor_cases.c.h" + #line 147 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -143,7 +155,7 @@ DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; - #line 147 "Python/executor_cases.c.h" + #line 159 "Python/executor_cases.c.h" Py_DECREF(value); stack_pointer[-1] = res; break; @@ -157,7 +169,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; - #line 161 "Python/executor_cases.c.h" + #line 173 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -174,12 +186,12 @@ } else { assert(Py_SIZE(value)); - #line 178 "Python/executor_cases.c.h" + #line 190 "Python/executor_cases.c.h" Py_DECREF(value); #line 354 "Python/bytecodes.c" res = Py_True; } - #line 183 "Python/executor_cases.c.h" + #line 195 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -193,11 +205,11 @@ assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); STAT_INC(TO_BOOL, hit); - #line 197 "Python/executor_cases.c.h" + #line 209 "Python/executor_cases.c.h" Py_DECREF(value); #line 364 "Python/bytecodes.c" res = Py_True; - #line 201 "Python/executor_cases.c.h" + #line 213 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -207,11 +219,11 @@ PyObject *res; #line 368 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 211 "Python/executor_cases.c.h" + #line 223 "Python/executor_cases.c.h" Py_DECREF(value); #line 370 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 215 "Python/executor_cases.c.h" + #line 227 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -222,7 +234,7 @@ #line 386 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - #line 226 "Python/executor_cases.c.h" + #line 238 "Python/executor_cases.c.h" break; } @@ -236,7 +248,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - #line 240 "Python/executor_cases.c.h" + #line 252 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -252,7 +264,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - #line 256 "Python/executor_cases.c.h" + #line 268 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -268,7 +280,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - #line 272 "Python/executor_cases.c.h" + #line 284 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -280,7 +292,7 @@ #line 422 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - #line 284 "Python/executor_cases.c.h" + #line 296 "Python/executor_cases.c.h" break; } @@ -294,7 +306,7 @@ ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - #line 298 "Python/executor_cases.c.h" + #line 310 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -310,7 +322,7 @@ ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - #line 314 "Python/executor_cases.c.h" + #line 326 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -326,7 +338,7 @@ ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - #line 330 "Python/executor_cases.c.h" + #line 342 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -338,7 +350,7 @@ #line 458 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - #line 342 "Python/executor_cases.c.h" + #line 354 "Python/executor_cases.c.h" break; } @@ -352,7 +364,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 356 "Python/executor_cases.c.h" + #line 368 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -376,7 +388,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 380 "Python/executor_cases.c.h" + #line 392 "Python/executor_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; break; @@ -400,7 +412,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 404 "Python/executor_cases.c.h" + #line 416 "Python/executor_cases.c.h" STACK_SHRINK(4); break; } @@ -423,7 +435,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 427 "Python/executor_cases.c.h" + #line 439 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -447,7 +459,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 451 "Python/executor_cases.c.h" + #line 463 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -465,14 +477,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 469 "Python/executor_cases.c.h" + #line 481 "Python/executor_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); #line 603 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 476 "Python/executor_cases.c.h" + #line 488 "Python/executor_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -485,7 +497,7 @@ PyObject *list = stack_pointer[-(2 + (oparg-1))]; #line 635 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 489 "Python/executor_cases.c.h" + #line 501 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -495,11 +507,11 @@ PyObject *set = stack_pointer[-(2 + (oparg-1))]; #line 639 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 499 "Python/executor_cases.c.h" + #line 511 "Python/executor_cases.c.h" Py_DECREF(v); #line 641 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 503 "Python/executor_cases.c.h" + #line 515 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -525,7 +537,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 529 "Python/executor_cases.c.h" + #line 541 "Python/executor_cases.c.h" STACK_SHRINK(3); break; } @@ -540,7 +552,7 @@ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 544 "Python/executor_cases.c.h" + #line 556 "Python/executor_cases.c.h" STACK_SHRINK(3); break; } @@ -551,12 +563,12 @@ #line 697 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 555 "Python/executor_cases.c.h" + #line 567 "Python/executor_cases.c.h" Py_DECREF(container); Py_DECREF(sub); #line 700 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 560 "Python/executor_cases.c.h" + #line 572 "Python/executor_cases.c.h" STACK_SHRINK(2); break; } @@ -567,11 +579,11 @@ #line 704 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 571 "Python/executor_cases.c.h" + #line 583 "Python/executor_cases.c.h" Py_DECREF(value); #line 707 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 575 "Python/executor_cases.c.h" + #line 587 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -583,12 +595,12 @@ #line 711 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 587 "Python/executor_cases.c.h" + #line 599 "Python/executor_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); #line 714 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 592 "Python/executor_cases.c.h" + #line 604 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -610,14 +622,14 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 614 "Python/executor_cases.c.h" + #line 626 "Python/executor_cases.c.h" Py_DECREF(obj); #line 832 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 621 "Python/executor_cases.c.h" + #line 633 "Python/executor_cases.c.h" Py_DECREF(obj); #line 837 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; @@ -632,7 +644,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 636 "Python/executor_cases.c.h" + #line 648 "Python/executor_cases.c.h" stack_pointer[-1] = iter; break; } @@ -683,7 +695,7 @@ Py_DECREF(next_iter); } } - #line 687 "Python/executor_cases.c.h" + #line 699 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; break; @@ -699,7 +711,7 @@ format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 703 "Python/executor_cases.c.h" + #line 715 "Python/executor_cases.c.h" Py_DECREF(iterable); #line 904 "Python/bytecodes.c" @@ -718,7 +730,7 @@ } if (iter == NULL) goto pop_1_error; - #line 722 "Python/executor_cases.c.h" + #line 734 "Python/executor_cases.c.h" stack_pointer[-1] = iter; break; } @@ -728,7 +740,7 @@ #line 1034 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 732 "Python/executor_cases.c.h" + #line 744 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -737,7 +749,7 @@ PyObject *value; #line 1085 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 741 "Python/executor_cases.c.h" + #line 753 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; break; @@ -767,7 +779,7 @@ if (true) goto error; } } - #line 771 "Python/executor_cases.c.h" + #line 783 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; break; @@ -782,7 +794,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 786 "Python/executor_cases.c.h" + #line 798 "Python/executor_cases.c.h" Py_DECREF(v); #line 1121 "Python/bytecodes.c" if (true) goto pop_1_error; @@ -791,11 +803,11 @@ err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 795 "Python/executor_cases.c.h" + #line 807 "Python/executor_cases.c.h" Py_DECREF(v); #line 1128 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 799 "Python/executor_cases.c.h" + #line 811 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -818,7 +830,7 @@ name); goto error; } - #line 822 "Python/executor_cases.c.h" + #line 834 "Python/executor_cases.c.h" break; } @@ -832,7 +844,7 @@ STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 836 "Python/executor_cases.c.h" + #line 848 "Python/executor_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -850,7 +862,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 854 "Python/executor_cases.c.h" + #line 866 "Python/executor_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -868,7 +880,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 872 "Python/executor_cases.c.h" + #line 884 "Python/executor_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -881,11 +893,11 @@ int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 885 "Python/executor_cases.c.h" + #line 897 "Python/executor_cases.c.h" Py_DECREF(seq); #line 1211 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 889 "Python/executor_cases.c.h" + #line 901 "Python/executor_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); break; } @@ -895,11 +907,11 @@ #line 1242 "Python/bytecodes.c" PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 899 "Python/executor_cases.c.h" + #line 911 "Python/executor_cases.c.h" Py_DECREF(owner); #line 1245 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 903 "Python/executor_cases.c.h" + #line 915 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -909,11 +921,11 @@ #line 1249 "Python/bytecodes.c" PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 913 "Python/executor_cases.c.h" + #line 925 "Python/executor_cases.c.h" Py_DECREF(v); #line 1252 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 917 "Python/executor_cases.c.h" + #line 929 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -931,7 +943,7 @@ } goto error; } - #line 935 "Python/executor_cases.c.h" + #line 947 "Python/executor_cases.c.h" break; } @@ -945,7 +957,7 @@ if (true) goto error; } Py_INCREF(locals); - #line 949 "Python/executor_cases.c.h" + #line 961 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = locals; break; @@ -1011,11 +1023,20 @@ } } } - #line 1015 "Python/executor_cases.c.h" + #line 1027 "Python/executor_cases.c.h" stack_pointer[-1] = v; break; } + case DELETE_FAST: { + #line 1435 "Python/bytecodes.c" + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); + #line 1037 "Python/executor_cases.c.h" + break; + } + case DELETE_DEREF: { #line 1452 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); @@ -1028,7 +1049,7 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1032 "Python/executor_cases.c.h" + #line 1053 "Python/executor_cases.c.h" break; } @@ -1070,7 +1091,7 @@ } Py_INCREF(value); } - #line 1074 "Python/executor_cases.c.h" + #line 1095 "Python/executor_cases.c.h" stack_pointer[-1] = value; break; } @@ -1085,7 +1106,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1089 "Python/executor_cases.c.h" + #line 1110 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; break; @@ -1098,7 +1119,7 @@ PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1102 "Python/executor_cases.c.h" + #line 1123 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -1115,7 +1136,7 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1119 "Python/executor_cases.c.h" + #line 1140 "Python/executor_cases.c.h" break; } @@ -1124,13 +1145,13 @@ PyObject *str; #line 1532 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1128 "Python/executor_cases.c.h" + #line 1149 "Python/executor_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } #line 1534 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1134 "Python/executor_cases.c.h" + #line 1155 "Python/executor_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1143,7 +1164,7 @@ #line 1538 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1147 "Python/executor_cases.c.h" + #line 1168 "Python/executor_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1156,7 +1177,7 @@ #line 1543 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1160 "Python/executor_cases.c.h" + #line 1181 "Python/executor_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1177,13 +1198,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1181 "Python/executor_cases.c.h" + #line 1202 "Python/executor_cases.c.h" Py_DECREF(iterable); #line 1559 "Python/bytecodes.c" if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); - #line 1187 "Python/executor_cases.c.h" + #line 1208 "Python/executor_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); break; @@ -1194,11 +1215,11 @@ PyObject *set = stack_pointer[-(2 + (oparg-1))]; #line 1566 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1198 "Python/executor_cases.c.h" + #line 1219 "Python/executor_cases.c.h" Py_DECREF(iterable); #line 1568 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1202 "Python/executor_cases.c.h" + #line 1223 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -1221,7 +1242,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 1225 "Python/executor_cases.c.h" + #line 1246 "Python/executor_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -1239,13 +1260,13 @@ if (map == NULL) goto error; - #line 1243 "Python/executor_cases.c.h" + #line 1264 "Python/executor_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } #line 1597 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 1249 "Python/executor_cases.c.h" + #line 1270 "Python/executor_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -1293,7 +1314,7 @@ Py_DECREF(ann_dict); } } - #line 1297 "Python/executor_cases.c.h" + #line 1318 "Python/executor_cases.c.h" break; } @@ -1311,14 +1332,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 1315 "Python/executor_cases.c.h" + #line 1336 "Python/executor_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); #line 1653 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 1322 "Python/executor_cases.c.h" + #line 1343 "Python/executor_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; break; @@ -1334,12 +1355,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 1338 "Python/executor_cases.c.h" + #line 1359 "Python/executor_cases.c.h" Py_DECREF(update); #line 1665 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 1343 "Python/executor_cases.c.h" + #line 1364 "Python/executor_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); break; @@ -1352,12 +1373,12 @@ if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 1356 "Python/executor_cases.c.h" + #line 1377 "Python/executor_cases.c.h" Py_DECREF(update); #line 1676 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 1361 "Python/executor_cases.c.h" + #line 1382 "Python/executor_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); break; @@ -1372,7 +1393,7 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 1376 "Python/executor_cases.c.h" + #line 1397 "Python/executor_cases.c.h" STACK_SHRINK(2); break; } @@ -1390,13 +1411,13 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - #line 1394 "Python/executor_cases.c.h" + #line 1415 "Python/executor_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); #line 1772 "Python/bytecodes.c" if (res == NULL) goto pop_3_error; - #line 1400 "Python/executor_cases.c.h" + #line 1421 "Python/executor_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1433,7 +1454,7 @@ res = res2; res2 = NULL; } - #line 1437 "Python/executor_cases.c.h" + #line 1458 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -1456,7 +1477,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - #line 1460 "Python/executor_cases.c.h" + #line 1481 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -1482,7 +1503,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - #line 1486 "Python/executor_cases.c.h" + #line 1507 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -1505,7 +1526,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - #line 1509 "Python/executor_cases.c.h" + #line 1530 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -1517,12 +1538,12 @@ PyObject *b; #line 2163 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 1521 "Python/executor_cases.c.h" + #line 1542 "Python/executor_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 2165 "Python/bytecodes.c" b = res ? Py_True : Py_False; - #line 1526 "Python/executor_cases.c.h" + #line 1547 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; break; @@ -1534,13 +1555,13 @@ PyObject *b; #line 2169 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 1538 "Python/executor_cases.c.h" + #line 1559 "Python/executor_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 2171 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - #line 1544 "Python/executor_cases.c.h" + #line 1565 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; break; @@ -1553,7 +1574,7 @@ PyObject *match; #line 2176 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 1557 "Python/executor_cases.c.h" + #line 1578 "Python/executor_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 2178 "Python/bytecodes.c" @@ -1564,7 +1585,7 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 1568 "Python/executor_cases.c.h" + #line 1589 "Python/executor_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 2186 "Python/bytecodes.c" @@ -1576,7 +1597,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 1580 "Python/executor_cases.c.h" + #line 1601 "Python/executor_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; break; @@ -1589,18 +1610,18 @@ #line 2197 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 1593 "Python/executor_cases.c.h" + #line 1614 "Python/executor_cases.c.h" Py_DECREF(right); #line 2200 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 1600 "Python/executor_cases.c.h" + #line 1621 "Python/executor_cases.c.h" Py_DECREF(right); #line 2205 "Python/bytecodes.c" b = res ? Py_True : Py_False; - #line 1604 "Python/executor_cases.c.h" + #line 1625 "Python/executor_cases.c.h" stack_pointer[-1] = b; break; } @@ -1614,7 +1635,7 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 1618 "Python/executor_cases.c.h" + #line 1639 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; break; @@ -1630,7 +1651,7 @@ // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 1634 "Python/executor_cases.c.h" + #line 1655 "Python/executor_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); @@ -1642,7 +1663,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_None; // Failure! } - #line 1646 "Python/executor_cases.c.h" + #line 1667 "Python/executor_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; break; @@ -1654,7 +1675,7 @@ #line 2327 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 1658 "Python/executor_cases.c.h" + #line 1679 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -1666,7 +1687,7 @@ #line 2332 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 1670 "Python/executor_cases.c.h" + #line 1691 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -1680,7 +1701,7 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 1684 "Python/executor_cases.c.h" + #line 1705 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; break; @@ -1692,11 +1713,11 @@ #line 2343 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 1696 "Python/executor_cases.c.h" + #line 1717 "Python/executor_cases.c.h" Py_DECREF(iterable); #line 2346 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 1700 "Python/executor_cases.c.h" + #line 1721 "Python/executor_cases.c.h" stack_pointer[-1] = iter; break; } @@ -1727,11 +1748,11 @@ if (iter == NULL) { goto error; } - #line 1731 "Python/executor_cases.c.h" + #line 1752 "Python/executor_cases.c.h" Py_DECREF(iterable); #line 2373 "Python/bytecodes.c" } - #line 1735 "Python/executor_cases.c.h" + #line 1756 "Python/executor_cases.c.h" stack_pointer[-1] = iter; break; } @@ -1762,7 +1783,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 1766 "Python/executor_cases.c.h" + #line 1787 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -1781,7 +1802,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 1785 "Python/executor_cases.c.h" + #line 1806 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -1798,7 +1819,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 1802 "Python/executor_cases.c.h" + #line 1823 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -1818,7 +1839,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 1822 "Python/executor_cases.c.h" + #line 1843 "Python/executor_cases.c.h" stack_pointer[-1] = func; break; } @@ -1851,7 +1872,7 @@ default: Py_UNREACHABLE(); } - #line 1855 "Python/executor_cases.c.h" + #line 1876 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; break; @@ -1864,13 +1885,13 @@ PyObject *slice; #line 3491 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 1868 "Python/executor_cases.c.h" + #line 1889 "Python/executor_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); #line 3493 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 1874 "Python/executor_cases.c.h" + #line 1895 "Python/executor_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -1887,7 +1908,7 @@ result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 1891 "Python/executor_cases.c.h" + #line 1912 "Python/executor_cases.c.h" stack_pointer[-1] = result; break; } @@ -1906,7 +1927,7 @@ else { res = value; } - #line 1910 "Python/executor_cases.c.h" + #line 1931 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -1920,7 +1941,7 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 1924 "Python/executor_cases.c.h" + #line 1945 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -1932,7 +1953,7 @@ #line 3526 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 1936 "Python/executor_cases.c.h" + #line 1957 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; break; @@ -1943,7 +1964,7 @@ PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; #line 3551 "Python/bytecodes.c" assert(oparg >= 2); - #line 1947 "Python/executor_cases.c.h" + #line 1968 "Python/executor_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; break; diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 6a42775e091208..ac3d800d8fe0fe 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -20,7 +20,7 @@ 0) #define EXIT_TRACE 300 -#define SET_IP 301 +#define SAVE_IP 301 #define _GUARD_BOTH_INT 302 #define _BINARY_OP_MULTIPLY_INT 303 #define _BINARY_OP_ADD_INT 304 @@ -1164,6 +1164,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { }; const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, + [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } }, @@ -1218,6 +1219,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, [LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } }, @@ -1266,7 +1268,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { #ifdef Py_DEBUG const char * const _PyOpcode_uop_name[512] = { [300] = "EXIT_TRACE", - [301] = "SET_IP", + [301] = "SAVE_IP", [302] = "_GUARD_BOTH_INT", [303] = "_BINARY_OP_MULTIPLY_INT", [304] = "_BINARY_OP_ADD_INT", diff --git a/Python/optimizer.c b/Python/optimizer.c index b00825ade16e07..32f0b1477d203c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -282,11 +282,6 @@ PyUnstable_Optimizer_NewCounter(void) ///////////////////// Experimental UOp Optimizer ///////////////////// -#ifdef Py_DEBUG - /* For debugging the interpreter: */ -# define LLTRACE 1 /* Low-level trace feature */ -#endif - static void uop_dealloc(_PyUOpExecutorObject *self) { PyObject_Free(self); @@ -308,60 +303,81 @@ translate_bytecode_to_trace( _PyUOpInstruction *trace, int max_length) { -#ifdef LLTRACE + int trace_length = 0; + +#ifdef Py_DEBUG char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); int lltrace = 0; if (uop_debug != NULL && *uop_debug >= '0') { lltrace = *uop_debug - '0'; // TODO: Parse an int and all that } - if (lltrace >= 4) { - fprintf(stderr, - "Optimizing %s (%s:%d) at offset %ld\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); - } -#define ADD_TO_TRACE(OPCODE, OPERAND) \ - if (lltrace >= 2) { \ - const char *opname = (OPCODE) < 256 ? _PyOpcode_OpName[(OPCODE)] : _PyOpcode_uop_name[(OPCODE)]; \ - fprintf(stderr, " ADD_TO_TRACE(%s, %" PRIu64 ")\n", opname, (uint64_t)(OPERAND)); \ - } \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].operand = (OPERAND); \ - trace_length++; +#define DPRINTF(level, ...) \ + if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } #else -#define ADD_TO_TRACE(OPCODE, OPERAND) \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].operand = (OPERAND); \ - trace_length++; +#define DPRINTF(level, ...) #endif - int trace_length = 0; - // Always reserve space for one uop, plus SET_UP, plus EXIT_TRACE - while (trace_length + 3 <= max_length) { +#define ADD_TO_TRACE(OPCODE, OPERAND) \ + DPRINTF(2, \ + " ADD_TO_TRACE(%s, %" PRIu64 ")\n", \ + (OPCODE) < 256 ? _PyOpcode_OpName[(OPCODE)] : _PyOpcode_uop_name[(OPCODE)], \ + (uint64_t)(OPERAND)); \ + assert(trace_length < max_length); \ + trace[trace_length].opcode = (OPCODE); \ + trace[trace_length].operand = (OPERAND); \ + trace_length++; + + DPRINTF(4, + "Optimizing %s (%s:%d) at offset %ld\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); + + for (;;) { + ADD_TO_TRACE(SAVE_IP, (int)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); int opcode = instr->op.code; uint64_t operand = instr->op.arg; switch (opcode) { case LOAD_FAST_LOAD_FAST: + case STORE_FAST_LOAD_FAST: + case STORE_FAST_STORE_FAST: { - // Reserve space for two uops (+ SETUP + EXIT_TRACE) + // Reserve space for two uops (+ SAVE_IP + EXIT_TRACE) if (trace_length + 4 > max_length) { + DPRINTF(1, "Ran out of space for LOAD_FAST_LOAD_FAST\n"); goto done; } uint64_t oparg1 = operand >> 4; uint64_t oparg2 = operand & 15; - ADD_TO_TRACE(LOAD_FAST, oparg1); - ADD_TO_TRACE(LOAD_FAST, oparg2); + switch (opcode) { + case LOAD_FAST_LOAD_FAST: + ADD_TO_TRACE(LOAD_FAST, oparg1); + ADD_TO_TRACE(LOAD_FAST, oparg2); + break; + case STORE_FAST_LOAD_FAST: + ADD_TO_TRACE(STORE_FAST, oparg1); + ADD_TO_TRACE(LOAD_FAST, oparg2); + break; + case STORE_FAST_STORE_FAST: + ADD_TO_TRACE(STORE_FAST, oparg1); + ADD_TO_TRACE(STORE_FAST, oparg2); + break; + default: + Py_FatalError("Missing case"); + } break; } default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; if (expansion->nuops > 0) { - // Reserve space for nuops (+ SETUP + EXIT_TRACE) + // Reserve space for nuops (+ SAVE_IP + EXIT_TRACE) int nuops = expansion->nuops; if (trace_length + nuops + 2 > max_length) { + DPRINTF(1, + "Ran out of space for %s\n", + opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]); goto done; } for (int i = 0; i < nuops; i++) { @@ -387,49 +403,45 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } ADD_TO_TRACE(expansion->uops[i].uop, operand); - assert(expansion->uops[0].size == 0); // TODO } break; } - // fprintf(stderr, "Unsupported opcode %d\n", opcode); - goto done; // Break out of while loop + DPRINTF(2, + "Unsupported opcode %s\n", + opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode]); + goto done; // Break out of loop } } instr++; // Add cache size for opcode instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - ADD_TO_TRACE(SET_IP, (int)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); } + done: - if (trace_length > 0) { + // Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE + if (trace_length > 3) { ADD_TO_TRACE(EXIT_TRACE, 0); -#ifdef LLTRACE - if (lltrace >= 1) { - fprintf(stderr, - "Created a trace for %s (%s:%d) at offset %ld -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive), - trace_length); - } -#endif + DPRINTF(1, + "Created a trace for %s (%s:%d) at offset %ld -- length %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive), + trace_length); + return trace_length; } else { -#ifdef LLTRACE - if (lltrace >= 4) { - fprintf(stderr, - "No trace for %s (%s:%d) at offset %ld\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); - } -#endif + DPRINTF(4, + "No trace for %s (%s:%d) at offset %ld\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + (long)(instr - (_Py_CODEUNIT *)code->co_code_adaptive)); } - return trace_length; + return 0; #undef ADD_TO_TRACE +#undef DPRINTF } static int diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 38be98034e0d83..657dfa93fd537d 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -308,8 +308,7 @@ class ActiveCacheEffect: FORBIDDEN_NAMES_IN_UOPS = ( - "resume_with_error", # Proxy for "goto", which isn't an IDENTIFIER - "unbound_local_error", + "resume_with_error", "kwnames", "next_instr", "oparg1", # Proxy for super-instructions like LOAD_FAST_LOAD_FAST @@ -401,20 +400,25 @@ def __init__(self, inst: parser.InstDef): def is_viable_uop(self) -> bool: """Whether this instruction is viable as a uop.""" if self.always_exits: + # print(f"Skipping {self.name} because it always exits") return False if self.instr_flags.HAS_ARG_FLAG: # If the instruction uses oparg, it cannot use any caches if self.active_caches: + # print(f"Skipping {self.name} because it uses oparg and caches") return False else: # If it doesn't use oparg, it can have one cache entry if len(self.active_caches) > 1: + # print(f"Skipping {self.name} because it has >1 cache entries") return False + res = True for forbidden in FORBIDDEN_NAMES_IN_UOPS: # TODO: Don't check in '#ifdef ENABLE_SPECIALIZATION' regions if variable_used(self.inst, forbidden): - return False - return True + # print(f"Skipping {self.name} because it uses {forbidden}") + res = False + return res def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: """Write one instruction, sans prologue and epilogue.""" @@ -1323,7 +1327,7 @@ def add(name: str) -> None: self.out.emit(make_text(name, counter)) counter += 1 add("EXIT_TRACE") - add("SET_IP") + add("SAVE_IP") for instr in self.instrs.values(): if instr.kind == "op" and instr.is_viable_uop(): add(instr.name) From bf06c6825cadeda54a9c0848fa51463a0e0b2cf8 Mon Sep 17 00:00:00 2001 From: Charlie Zhao Date: Tue, 4 Jul 2023 04:10:01 +0800 Subject: [PATCH 28/31] gh-106078: Move `context template` to decimal module global state (#106346) --- Modules/_decimal/_decimal.c | 59 +++++++++++---------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 -- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index da623725003428..6da5095b91018a 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -49,6 +49,14 @@ typedef struct { /* Top level Exception; inherits from ArithmeticError */ PyObject *DecimalException; + + /* Template for creating new thread contexts, calling Context() without + * arguments and initializing the module_context on first access. */ + PyObject *default_context_template; + + /* Basic and extended context templates */ + PyObject *basic_context_template; + PyObject *extended_context_template; } decimal_state; static decimal_state global_state; @@ -147,14 +155,6 @@ static PyDecContextObject *cached_context = NULL; static PyObject *current_context_var = NULL; #endif -/* Template for creating new thread contexts, calling Context() without - * arguments and initializing the module_context on first access. */ -static PyObject *default_context_template = NULL; -/* Basic and extended context templates */ -static PyObject *basic_context_template = NULL; -static PyObject *extended_context_template = NULL; - - /* Error codes for functions that return signals or conditions */ #define DEC_INVALID_SIGNALS (MPD_Max_status+1U) #define DEC_ERR_OCCURRED (DEC_INVALID_SIGNALS<<1) @@ -1272,8 +1272,8 @@ context_new(PyTypeObject *type, PyObject *args UNUSED, PyObject *kwds UNUSED) ctx = CTX(self); - if (default_context_template) { - *ctx = *CTX(default_context_template); + if (state->default_context_template) { + *ctx = *CTX(state->default_context_template); } else { *ctx = dflt_ctx; @@ -1576,7 +1576,7 @@ current_context_from_dict(void) } /* Set up a new thread local context. */ - tl_context = context_copy(default_context_template, NULL); + tl_context = context_copy(state->default_context_template, NULL); if (tl_context == NULL) { return NULL; } @@ -1649,9 +1649,9 @@ PyDec_SetCurrentContext(PyObject *self UNUSED, PyObject *v) /* If the new context is one of the templates, make a copy. * This is the current behavior of decimal.py. */ - if (v == default_context_template || - v == basic_context_template || - v == extended_context_template) { + if (v == state->default_context_template || + v == state->basic_context_template || + v == state->extended_context_template) { v = context_copy(v, NULL); if (v == NULL) { return NULL; @@ -1675,7 +1675,8 @@ PyDec_SetCurrentContext(PyObject *self UNUSED, PyObject *v) static PyObject * init_current_context(void) { - PyObject *tl_context = context_copy(default_context_template, NULL); + decimal_state *state = GLOBAL_STATE(); + PyObject *tl_context = context_copy(state->default_context_template, NULL); if (tl_context == NULL) { return NULL; } @@ -1730,9 +1731,9 @@ PyDec_SetCurrentContext(PyObject *self UNUSED, PyObject *v) /* If the new context is one of the templates, make a copy. * This is the current behavior of decimal.py. */ - if (v == default_context_template || - v == basic_context_template || - v == extended_context_template) { + if (v == state->default_context_template || + v == state->basic_context_template || + v == state->extended_context_template) { v = context_copy(v, NULL); if (v == NULL) { return NULL; @@ -5980,10 +5981,10 @@ PyInit__decimal(void) /* Init default context template first */ - ASSIGN_PTR(default_context_template, + ASSIGN_PTR(state->default_context_template, PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL)); CHECK_INT(PyModule_AddObject(m, "DefaultContext", - Py_NewRef(default_context_template))); + Py_NewRef(state->default_context_template))); #ifndef WITH_DECIMAL_CONTEXTVAR ASSIGN_PTR(tls_context_key, PyUnicode_FromString("___DECIMAL_CTX__")); @@ -5995,18 +5996,18 @@ PyInit__decimal(void) CHECK_INT(PyModule_AddObject(m, "HAVE_THREADS", Py_NewRef(Py_True))); /* Init basic context template */ - ASSIGN_PTR(basic_context_template, + ASSIGN_PTR(state->basic_context_template, PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL)); - init_basic_context(basic_context_template); + init_basic_context(state->basic_context_template); CHECK_INT(PyModule_AddObject(m, "BasicContext", - Py_NewRef(basic_context_template))); + Py_NewRef(state->basic_context_template))); /* Init extended context template */ - ASSIGN_PTR(extended_context_template, + ASSIGN_PTR(state->extended_context_template, PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL)); - init_extended_context(extended_context_template); + init_extended_context(state->extended_context_template); CHECK_INT(PyModule_AddObject(m, "ExtendedContext", - Py_NewRef(extended_context_template))); + Py_NewRef(state->extended_context_template))); /* Init mpd_ssize_t constants */ @@ -6046,14 +6047,14 @@ PyInit__decimal(void) Py_CLEAR(MutableMapping); /* GCOV_NOT_REACHED */ Py_CLEAR(SignalTuple); /* GCOV_NOT_REACHED */ Py_CLEAR(state->DecimalTuple); /* GCOV_NOT_REACHED */ - Py_CLEAR(default_context_template); /* GCOV_NOT_REACHED */ + Py_CLEAR(state->default_context_template); /* GCOV_NOT_REACHED */ #ifndef WITH_DECIMAL_CONTEXTVAR Py_CLEAR(tls_context_key); /* GCOV_NOT_REACHED */ #else Py_CLEAR(current_context_var); /* GCOV_NOT_REACHED */ #endif - Py_CLEAR(basic_context_template); /* GCOV_NOT_REACHED */ - Py_CLEAR(extended_context_template); /* GCOV_NOT_REACHED */ + Py_CLEAR(state->basic_context_template); /* GCOV_NOT_REACHED */ + Py_CLEAR(state->extended_context_template); /* GCOV_NOT_REACHED */ Py_CLEAR(m); /* GCOV_NOT_REACHED */ return NULL; /* GCOV_NOT_REACHED */ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 8fdc54df2b0722..ad3d9b6513d69c 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -422,10 +422,7 @@ Modules/_datetimemodule.c - us_per_day - Modules/_datetimemodule.c - us_per_week - Modules/_datetimemodule.c - seconds_per_day - Modules/_decimal/_decimal.c - global_state - -Modules/_decimal/_decimal.c - basic_context_template - Modules/_decimal/_decimal.c - current_context_var - -Modules/_decimal/_decimal.c - default_context_template - -Modules/_decimal/_decimal.c - extended_context_template - Modules/_decimal/_decimal.c - round_map - Modules/_decimal/_decimal.c - Rational - Modules/_decimal/_decimal.c - SignalTuple - From 7f4c8121db62a9f72f00f2d9f73381e82f289581 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jul 2023 22:16:50 +0200 Subject: [PATCH 29/31] gh-106368: Increase Argument Clinic test coverage (#106369) Add tests for 'self' and 'defining_class' converter requirements. --- Lib/test/test_clinic.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index fab2d2bb0bdbe4..51d2ac972752fd 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -829,6 +829,63 @@ def test_kwarg_splats_disallowed_in_function_call_annotations(self): out = self.parse_function_should_fail(fn) self.assertEqual(out, expected_error_msg) + def test_self_param_placement(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'self' parameter, if specified, must be the very first thing " + "in the parameter block.\n" + ) + block = """ + module foo + foo.func + a: int + self: self(type="PyObject *") + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + + def test_self_param_cannot_be_optional(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'self' parameter cannot be marked optional.\n" + ) + block = """ + module foo + foo.func + self: self(type="PyObject *") = None + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + + def test_defining_class_param_placement(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'defining_class' parameter, if specified, must either be the " + "first thing in the parameter block, or come just after 'self'.\n" + ) + block = """ + module foo + foo.func + self: self(type="PyObject *") + a: int + cls: defining_class + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + + def test_defining_class_param_cannot_be_optional(self): + expected_error_msg = ( + "Error on line 0:\n" + "A 'defining_class' parameter cannot be marked optional.\n" + ) + block = """ + module foo + foo.func + cls: defining_class(type="PyObject *") = None + """ + out = self.parse_function_should_fail(block) + self.assertEqual(out, expected_error_msg) + def test_unused_param(self): block = self.parse(""" module foo From e5862113dde7a66b08f1ece542a3cfaf0a3d9080 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Jul 2023 21:28:27 +0100 Subject: [PATCH 30/31] GH-104584: Fix ENTER_EXECUTOR (GH-106141) * Check eval-breaker in ENTER_EXECUTOR. * Make sure that frame->prev_instr is set before entering executor. --- Include/internal/pycore_ceval.h | 2 + Python/bytecodes.c | 19 +- Python/ceval.c | 70 +------ Python/ceval_gil.c | 61 +++++- Python/ceval_macros.h | 4 +- Python/executor_cases.c.h | 44 ++--- Python/generated_cases.c.h | 339 ++++++++++++++++---------------- Python/opcode_metadata.h | 2 +- 8 files changed, 272 insertions(+), 269 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 9e9b523e7c2222..46bc18cff86d5a 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -152,6 +152,8 @@ extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); extern PyObject* _Py_MakeCoro(PyFunctionObject *func); +/* Handle signals, pending calls, GIL drop request + and asynchronous exception */ extern int _Py_HandlePending(PyThreadState *tstate); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8aab7ebc5cf0cb..f69ac2beef4c20 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -141,8 +141,8 @@ dummy_func( ERROR_IF(err, error); next_instr--; } - else if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { - goto handle_eval_breaker; + else if (oparg < 2) { + CHECK_EVAL_BREAKER(); } } @@ -158,6 +158,9 @@ dummy_func( next_instr--; } else { + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); @@ -168,9 +171,6 @@ dummy_func( next_instr = frame->prev_instr; DISPATCH(); } - if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { - goto handle_eval_breaker; - } } } @@ -2223,6 +2223,7 @@ dummy_func( } inst(JUMP_BACKWARD, (--)) { + CHECK_EVAL_BREAKER(); _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); @@ -2240,7 +2241,6 @@ dummy_func( goto resume_frame; } #endif /* ENABLE_SPECIALIZATION */ - CHECK_EVAL_BREAKER(); } pseudo(JUMP) = { @@ -2254,8 +2254,13 @@ dummy_func( }; inst(ENTER_EXECUTOR, (--)) { + CHECK_EVAL_BREAKER(); + PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); + frame->prev_instr = next_instr - 1; Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { @@ -3570,8 +3575,8 @@ dummy_func( } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { diff --git a/Python/ceval.c b/Python/ceval.c index 6ce1a6a636ed71..9bcb83f9c993cf 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -763,68 +763,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); -handle_eval_breaker: - - /* Do periodic things, like check for signals and async I/0. - * We need to do reasonably frequently, but not too frequently. - * All loops should include a check of the eval breaker. - * We also check on return from any builtin function. - * - * ## More Details ### - * - * The eval loop (this function) normally executes the instructions - * of a code object sequentially. However, the runtime supports a - * number of out-of-band execution scenarios that may pause that - * sequential execution long enough to do that out-of-band work - * in the current thread using the current PyThreadState. - * - * The scenarios include: - * - * - cyclic garbage collection - * - GIL drop requests - * - "async" exceptions - * - "pending calls" (some only in the main thread) - * - signal handling (only in the main thread) - * - * When the need for one of the above is detected, the eval loop - * pauses long enough to handle the detected case. Then, if doing - * so didn't trigger an exception, the eval loop resumes executing - * the sequential instructions. - * - * To make this work, the eval loop periodically checks if any - * of the above needs to happen. The individual checks can be - * expensive if computed each time, so a while back we switched - * to using pre-computed, per-interpreter variables for the checks, - * and later consolidated that to a single "eval breaker" variable - * (now a PyInterpreterState field). - * - * For the longest time, the eval breaker check would happen - * frequently, every 5 or so times through the loop, regardless - * of what instruction ran last or what would run next. Then, in - * early 2021 (gh-18334, commit 4958f5d), we switched to checking - * the eval breaker less frequently, by hard-coding the check to - * specific places in the eval loop (e.g. certain instructions). - * The intent then was to check after returning from calls - * and on the back edges of loops. - * - * In addition to being more efficient, that approach keeps - * the eval loop from running arbitrary code between instructions - * that don't handle that well. (See gh-74174.) - * - * Currently, the eval breaker check happens here at the - * "handle_eval_breaker" label. Some instructions come here - * explicitly (goto) and some indirectly. Notably, the check - * happens on back edges in the control flow graph, which - * pretty much applies to all loops and most calls. - * (See bytecodes.c for exact information.) - * - * One consequence of this approach is that it might not be obvious - * how to force any specific thread to pick up the eval breaker, - * or for any specific thread to not pick it up. Mostly this - * involves judicious uses of locks and careful ordering of code, - * while avoiding code that might trigger the eval breaker - * until so desired. - */ if (_Py_HandlePending(tstate) != 0) { goto error; } @@ -2796,13 +2734,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject PyThreadState *tstate = _PyThreadState_GET(); _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor; - // Equivalent to CHECK_EVAL_BREAKER() - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)) { - if (_Py_HandlePending(tstate) != 0) { - goto error; - } - } + CHECK_EVAL_BREAKER(); OBJECT_STAT_INC(optimization_traces_executed); _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index c6b1f9ef689ee1..7c9ad07cc7207b 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1052,8 +1052,65 @@ _PyEval_FiniState(struct _ceval_state *ceval) } } -/* Handle signals, pending calls, GIL drop request - and asynchronous exception */ + +/* Do periodic things, like check for signals and async I/0. +* We need to do reasonably frequently, but not too frequently. +* All loops should include a check of the eval breaker. +* We also check on return from any builtin function. +* +* ## More Details ### +* +* The eval loop (this function) normally executes the instructions +* of a code object sequentially. However, the runtime supports a +* number of out-of-band execution scenarios that may pause that +* sequential execution long enough to do that out-of-band work +* in the current thread using the current PyThreadState. +* +* The scenarios include: +* +* - cyclic garbage collection +* - GIL drop requests +* - "async" exceptions +* - "pending calls" (some only in the main thread) +* - signal handling (only in the main thread) +* +* When the need for one of the above is detected, the eval loop +* pauses long enough to handle the detected case. Then, if doing +* so didn't trigger an exception, the eval loop resumes executing +* the sequential instructions. +* +* To make this work, the eval loop periodically checks if any +* of the above needs to happen. The individual checks can be +* expensive if computed each time, so a while back we switched +* to using pre-computed, per-interpreter variables for the checks, +* and later consolidated that to a single "eval breaker" variable +* (now a PyInterpreterState field). +* +* For the longest time, the eval breaker check would happen +* frequently, every 5 or so times through the loop, regardless +* of what instruction ran last or what would run next. Then, in +* early 2021 (gh-18334, commit 4958f5d), we switched to checking +* the eval breaker less frequently, by hard-coding the check to +* specific places in the eval loop (e.g. certain instructions). +* The intent then was to check after returning from calls +* and on the back edges of loops. +* +* In addition to being more efficient, that approach keeps +* the eval loop from running arbitrary code between instructions +* that don't handle that well. (See gh-74174.) +* +* Currently, the eval breaker check happens on back edges in +* the control flow graph, which pretty much applies to all loops, +* and most calls. +* (See bytecodes.c for exact information.) +* +* One consequence of this approach is that it might not be obvious +* how to force any specific thread to pick up the eval breaker, +* or for any specific thread to not pick it up. Mostly this +* involves judicious uses of locks and careful ordering of code, +* while avoiding code that might trigger the eval breaker +* until so desired. +*/ int _Py_HandlePending(PyThreadState *tstate) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0d41ef5a14cef4..72800aaaaa2ac4 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -117,7 +117,9 @@ #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)) { \ - goto handle_eval_breaker; \ + if (_Py_HandlePending(tstate) != 0) { \ + goto error; \ + } \ } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d1e0443db613d2..39a4490e51c24a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1629,7 +1629,7 @@ case GET_LEN: { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2304 "Python/bytecodes.c" + #line 2309 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -1646,7 +1646,7 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2312 "Python/bytecodes.c" + #line 2317 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -1655,7 +1655,7 @@ Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2317 "Python/bytecodes.c" + #line 2322 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -1672,7 +1672,7 @@ case MATCH_MAPPING: { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2327 "Python/bytecodes.c" + #line 2332 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; #line 1679 "Python/executor_cases.c.h" @@ -1684,7 +1684,7 @@ case MATCH_SEQUENCE: { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2332 "Python/bytecodes.c" + #line 2337 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; #line 1691 "Python/executor_cases.c.h" @@ -1697,7 +1697,7 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2337 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -1710,12 +1710,12 @@ case GET_ITER: { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2343 "Python/bytecodes.c" + #line 2348 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); #line 1717 "Python/executor_cases.c.h" Py_DECREF(iterable); - #line 2346 "Python/bytecodes.c" + #line 2351 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; #line 1721 "Python/executor_cases.c.h" stack_pointer[-1] = iter; @@ -1725,7 +1725,7 @@ case GET_YIELD_FROM_ITER: { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2350 "Python/bytecodes.c" + #line 2355 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -1750,7 +1750,7 @@ } #line 1752 "Python/executor_cases.c.h" Py_DECREF(iterable); - #line 2373 "Python/bytecodes.c" + #line 2378 "Python/bytecodes.c" } #line 1756 "Python/executor_cases.c.h" stack_pointer[-1] = iter; @@ -1762,7 +1762,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2605 "Python/bytecodes.c" + #line 2610 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -1792,7 +1792,7 @@ case PUSH_EXC_INFO: { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2644 "Python/bytecodes.c" + #line 2649 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -1811,7 +1811,7 @@ case EXIT_INIT_CHECK: { PyObject *should_be_none = stack_pointer[-1]; - #line 3013 "Python/bytecodes.c" + #line 3018 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -1827,7 +1827,7 @@ case MAKE_FUNCTION: { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3427 "Python/bytecodes.c" + #line 3432 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -1847,7 +1847,7 @@ case SET_FUNCTION_ATTRIBUTE: { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3441 "Python/bytecodes.c" + #line 3446 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -1883,13 +1883,13 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3491 "Python/bytecodes.c" + #line 3496 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); #line 1889 "Python/executor_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3493 "Python/bytecodes.c" + #line 3498 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } #line 1895 "Python/executor_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); @@ -1901,7 +1901,7 @@ case CONVERT_VALUE: { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3497 "Python/bytecodes.c" + #line 3502 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -1916,7 +1916,7 @@ case FORMAT_SIMPLE: { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3506 "Python/bytecodes.c" + #line 3511 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -1936,7 +1936,7 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3519 "Python/bytecodes.c" + #line 3524 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -1950,7 +1950,7 @@ case COPY: { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3526 "Python/bytecodes.c" + #line 3531 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); #line 1957 "Python/executor_cases.c.h" @@ -1962,7 +1962,7 @@ case SWAP: { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3551 "Python/bytecodes.c" + #line 3556 "Python/bytecodes.c" assert(oparg >= 2); #line 1968 "Python/executor_cases.c.h" stack_pointer[-1] = bottom; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 21cd2d0126ab30..eb2422943984b1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -17,8 +17,8 @@ if (err) goto error; next_instr--; } - else if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { - goto handle_eval_breaker; + else if (oparg < 2) { + CHECK_EVAL_BREAKER(); } #line 24 "Python/generated_cases.c.h" DISPATCH(); @@ -37,6 +37,9 @@ next_instr--; } else { + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); @@ -47,9 +50,6 @@ next_instr = frame->prev_instr; DISPATCH(); } - if (_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker) && oparg < 2) { - goto handle_eval_breaker; - } } #line 55 "Python/generated_cases.c.h" DISPATCH(); @@ -3182,6 +3182,7 @@ TARGET(JUMP_BACKWARD) { #line 2226 "Python/bytecodes.c" + CHECK_EVAL_BREAKER(); _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); @@ -3199,15 +3200,19 @@ goto resume_frame; } #endif /* ENABLE_SPECIALIZATION */ - #line 3203 "Python/generated_cases.c.h" - CHECK_EVAL_BREAKER(); + #line 3204 "Python/generated_cases.c.h" DISPATCH(); } TARGET(ENTER_EXECUTOR) { #line 2257 "Python/bytecodes.c" + CHECK_EVAL_BREAKER(); + PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); + frame->prev_instr = next_instr - 1; Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { @@ -3215,81 +3220,81 @@ goto resume_with_error; } goto resume_frame; - #line 3219 "Python/generated_cases.c.h" + #line 3224 "Python/generated_cases.c.h" } TARGET(POP_JUMP_IF_FALSE) { PyObject *cond = stack_pointer[-1]; - #line 2269 "Python/bytecodes.c" + #line 2274 "Python/bytecodes.c" assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); - #line 3227 "Python/generated_cases.c.h" + #line 3232 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2274 "Python/bytecodes.c" + #line 2279 "Python/bytecodes.c" assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); - #line 3237 "Python/generated_cases.c.h" + #line 3242 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2279 "Python/bytecodes.c" + #line 2284 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 3246 "Python/generated_cases.c.h" + #line 3251 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2281 "Python/bytecodes.c" + #line 2286 "Python/bytecodes.c" JUMPBY(oparg); } - #line 3251 "Python/generated_cases.c.h" + #line 3256 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2286 "Python/bytecodes.c" + #line 2291 "Python/bytecodes.c" if (Py_IsNone(value)) { JUMPBY(oparg); } else { - #line 3263 "Python/generated_cases.c.h" + #line 3268 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2291 "Python/bytecodes.c" + #line 2296 "Python/bytecodes.c" } - #line 3267 "Python/generated_cases.c.h" + #line 3272 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2295 "Python/bytecodes.c" + #line 2300 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 3280 "Python/generated_cases.c.h" + #line 3285 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2304 "Python/bytecodes.c" + #line 2309 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3293 "Python/generated_cases.c.h" + #line 3298 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3300,16 +3305,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2312 "Python/bytecodes.c" + #line 2317 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3309 "Python/generated_cases.c.h" + #line 3314 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2317 "Python/bytecodes.c" + #line 2322 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3317,7 +3322,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_None; // Failure! } - #line 3321 "Python/generated_cases.c.h" + #line 3326 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3326,10 +3331,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2327 "Python/bytecodes.c" + #line 2332 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 3333 "Python/generated_cases.c.h" + #line 3338 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3338,10 +3343,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2332 "Python/bytecodes.c" + #line 2337 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 3345 "Python/generated_cases.c.h" + #line 3350 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3351,11 +3356,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2337 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3359 "Python/generated_cases.c.h" + #line 3364 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3364,14 +3369,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2343 "Python/bytecodes.c" + #line 2348 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3371 "Python/generated_cases.c.h" + #line 3376 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2346 "Python/bytecodes.c" + #line 2351 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3375 "Python/generated_cases.c.h" + #line 3380 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3379,7 +3384,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2350 "Python/bytecodes.c" + #line 2355 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3402,11 +3407,11 @@ if (iter == NULL) { goto error; } - #line 3406 "Python/generated_cases.c.h" + #line 3411 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2373 "Python/bytecodes.c" + #line 2378 "Python/bytecodes.c" } - #line 3410 "Python/generated_cases.c.h" + #line 3415 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3416,7 +3421,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2391 "Python/bytecodes.c" + #line 2396 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3448,7 +3453,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3452 "Python/generated_cases.c.h" + #line 3457 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3456,7 +3461,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2425 "Python/bytecodes.c" + #line 2430 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3482,14 +3487,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3486 "Python/generated_cases.c.h" + #line 3491 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2453 "Python/bytecodes.c" + #line 2458 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3510,7 +3515,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3514 "Python/generated_cases.c.h" + #line 3519 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3520,7 +3525,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2476 "Python/bytecodes.c" + #line 2481 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3541,7 +3546,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3545 "Python/generated_cases.c.h" + #line 3550 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3551,7 +3556,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2499 "Python/bytecodes.c" + #line 2504 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3570,7 +3575,7 @@ if (next == NULL) { goto error; } - #line 3574 "Python/generated_cases.c.h" + #line 3579 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3579,7 +3584,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2520 "Python/bytecodes.c" + #line 2525 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3595,14 +3600,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3599 "Python/generated_cases.c.h" + #line 3604 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2538 "Python/bytecodes.c" + #line 2543 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3625,16 +3630,16 @@ Py_DECREF(enter); goto error; } - #line 3629 "Python/generated_cases.c.h" + #line 3634 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2561 "Python/bytecodes.c" + #line 2566 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3638 "Python/generated_cases.c.h" + #line 3643 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3645,7 +3650,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2570 "Python/bytecodes.c" + #line 2575 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3671,16 +3676,16 @@ Py_DECREF(enter); goto error; } - #line 3675 "Python/generated_cases.c.h" + #line 3680 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2596 "Python/bytecodes.c" + #line 2601 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3684 "Python/generated_cases.c.h" + #line 3689 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3692,7 +3697,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2605 "Python/bytecodes.c" + #line 2610 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3713,7 +3718,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3717 "Python/generated_cases.c.h" + #line 3722 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3722,7 +3727,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2644 "Python/bytecodes.c" + #line 2649 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3732,7 +3737,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3736 "Python/generated_cases.c.h" + #line 3741 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3746,7 +3751,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2656 "Python/bytecodes.c" + #line 2661 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3763,7 +3768,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3767 "Python/generated_cases.c.h" + #line 3772 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3777,7 +3782,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2675 "Python/bytecodes.c" + #line 2680 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3787,7 +3792,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3791 "Python/generated_cases.c.h" + #line 3796 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3801,7 +3806,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2687 "Python/bytecodes.c" + #line 2692 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3815,7 +3820,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3819 "Python/generated_cases.c.h" + #line 3824 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3824,16 +3829,16 @@ } TARGET(KW_NAMES) { - #line 2703 "Python/bytecodes.c" + #line 2708 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(FRAME_CO_CONSTS)); kwnames = GETITEM(FRAME_CO_CONSTS, oparg); - #line 3832 "Python/generated_cases.c.h" + #line 3837 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2709 "Python/bytecodes.c" + #line 2714 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3846,7 +3851,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3850 "Python/generated_cases.c.h" + #line 3855 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3856,7 +3861,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2755 "Python/bytecodes.c" + #line 2760 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3938,7 +3943,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3942 "Python/generated_cases.c.h" + #line 3947 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3950,7 +3955,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2843 "Python/bytecodes.c" + #line 2848 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3960,7 +3965,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3964 "Python/generated_cases.c.h" + #line 3969 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3969,7 +3974,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2855 "Python/bytecodes.c" + #line 2860 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3995,7 +4000,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3999 "Python/generated_cases.c.h" + #line 4004 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -4003,7 +4008,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2883 "Python/bytecodes.c" + #line 2888 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -4039,7 +4044,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 4043 "Python/generated_cases.c.h" + #line 4048 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -4047,7 +4052,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2921 "Python/bytecodes.c" + #line 2926 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4057,7 +4062,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 4061 "Python/generated_cases.c.h" + #line 4066 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4070,7 +4075,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2933 "Python/bytecodes.c" + #line 2938 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4081,7 +4086,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4085 "Python/generated_cases.c.h" + #line 4090 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4095,7 +4100,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2947 "Python/bytecodes.c" + #line 2952 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4106,7 +4111,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4110 "Python/generated_cases.c.h" + #line 4115 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4119,7 +4124,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; - #line 2961 "Python/bytecodes.c" + #line 2966 "Python/bytecodes.c" /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -4169,12 +4174,12 @@ frame = cframe.current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); goto start_frame; - #line 4173 "Python/generated_cases.c.h" + #line 4178 "Python/generated_cases.c.h" } TARGET(EXIT_INIT_CHECK) { PyObject *should_be_none = stack_pointer[-1]; - #line 3013 "Python/bytecodes.c" + #line 3018 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -4182,7 +4187,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 4186 "Python/generated_cases.c.h" + #line 4191 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -4192,7 +4197,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3023 "Python/bytecodes.c" + #line 3028 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4214,7 +4219,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4218 "Python/generated_cases.c.h" + #line 4223 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4228,7 +4233,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3048 "Python/bytecodes.c" + #line 3053 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4256,7 +4261,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4260 "Python/generated_cases.c.h" + #line 4265 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4270,7 +4275,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3079 "Python/bytecodes.c" + #line 3084 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4302,7 +4307,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4306 "Python/generated_cases.c.h" + #line 4311 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4316,7 +4321,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3114 "Python/bytecodes.c" + #line 3119 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4348,7 +4353,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4352 "Python/generated_cases.c.h" + #line 4357 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4362,7 +4367,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3149 "Python/bytecodes.c" + #line 3154 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4387,7 +4392,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4391 "Python/generated_cases.c.h" + #line 4396 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4400,7 +4405,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3176 "Python/bytecodes.c" + #line 3181 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4427,7 +4432,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4431 "Python/generated_cases.c.h" + #line 4436 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4439,7 +4444,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3206 "Python/bytecodes.c" + #line 3211 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4457,14 +4462,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4461 "Python/generated_cases.c.h" + #line 4466 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3226 "Python/bytecodes.c" + #line 3231 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4495,7 +4500,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4499 "Python/generated_cases.c.h" + #line 4504 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4508,7 +4513,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3260 "Python/bytecodes.c" + #line 3265 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4537,7 +4542,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4541 "Python/generated_cases.c.h" + #line 4546 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4550,7 +4555,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3292 "Python/bytecodes.c" + #line 3297 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4579,7 +4584,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4583 "Python/generated_cases.c.h" + #line 4588 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4592,7 +4597,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3324 "Python/bytecodes.c" + #line 3329 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4620,7 +4625,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4624 "Python/generated_cases.c.h" + #line 4629 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4630,9 +4635,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3355 "Python/bytecodes.c" + #line 3360 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4636 "Python/generated_cases.c.h" + #line 4641 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4641,7 +4646,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3359 "Python/bytecodes.c" + #line 3364 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4703,14 +4708,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4707 "Python/generated_cases.c.h" + #line 4712 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3421 "Python/bytecodes.c" + #line 3426 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4714 "Python/generated_cases.c.h" + #line 4719 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4721,7 +4726,7 @@ TARGET(MAKE_FUNCTION) { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3427 "Python/bytecodes.c" + #line 3432 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4733,7 +4738,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4737 "Python/generated_cases.c.h" + #line 4742 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4741,7 +4746,7 @@ TARGET(SET_FUNCTION_ATTRIBUTE) { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3441 "Python/bytecodes.c" + #line 3446 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4766,14 +4771,14 @@ default: Py_UNREACHABLE(); } - #line 4770 "Python/generated_cases.c.h" + #line 4775 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3468 "Python/bytecodes.c" + #line 3473 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4794,7 +4799,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4798 "Python/generated_cases.c.h" + #line 4803 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4802,15 +4807,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3491 "Python/bytecodes.c" + #line 3496 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4808 "Python/generated_cases.c.h" + #line 4813 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3493 "Python/bytecodes.c" + #line 3498 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4814 "Python/generated_cases.c.h" + #line 4819 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4820,14 +4825,14 @@ TARGET(CONVERT_VALUE) { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3497 "Python/bytecodes.c" + #line 3502 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4831 "Python/generated_cases.c.h" + #line 4836 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4835,7 +4840,7 @@ TARGET(FORMAT_SIMPLE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3506 "Python/bytecodes.c" + #line 3511 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4846,7 +4851,7 @@ else { res = value; } - #line 4850 "Python/generated_cases.c.h" + #line 4855 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4855,12 +4860,12 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3519 "Python/bytecodes.c" + #line 3524 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4864 "Python/generated_cases.c.h" + #line 4869 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4869,10 +4874,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3526 "Python/bytecodes.c" + #line 3531 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4876 "Python/generated_cases.c.h" + #line 4881 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4884,7 +4889,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3531 "Python/bytecodes.c" + #line 3536 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4899,12 +4904,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4903 "Python/generated_cases.c.h" + #line 4908 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3546 "Python/bytecodes.c" + #line 3551 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4908 "Python/generated_cases.c.h" + #line 4913 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4914,16 +4919,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3551 "Python/bytecodes.c" + #line 3556 "Python/bytecodes.c" assert(oparg >= 2); - #line 4920 "Python/generated_cases.c.h" + #line 4925 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3555 "Python/bytecodes.c" + #line 3560 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4935,48 +4940,48 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4939 "Python/generated_cases.c.h" + #line 4944 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3569 "Python/bytecodes.c" + #line 3574 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4945 "Python/generated_cases.c.h" + #line 4950 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3573 "Python/bytecodes.c" - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4952 "Python/generated_cases.c.h" + #line 3578 "Python/bytecodes.c" CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + #line 4958 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3578 "Python/bytecodes.c" + #line 3583 "Python/bytecodes.c" PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; int offset = Py_IsTrue(cond) * oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4964 "Python/generated_cases.c.h" + #line 4969 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3586 "Python/bytecodes.c" + #line 3591 "Python/bytecodes.c" PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; int offset = Py_IsFalse(cond) * oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4975 "Python/generated_cases.c.h" + #line 4980 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3594 "Python/bytecodes.c" + #line 3599 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4988,12 +4993,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4992 "Python/generated_cases.c.h" + #line 4997 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3608 "Python/bytecodes.c" + #line 3613 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -5005,30 +5010,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 5009 "Python/generated_cases.c.h" + #line 5014 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3622 "Python/bytecodes.c" + #line 3627 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 5020 "Python/generated_cases.c.h" + #line 5025 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3630 "Python/bytecodes.c" + #line 3635 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 5027 "Python/generated_cases.c.h" + #line 5032 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3635 "Python/bytecodes.c" + #line 3640 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 5034 "Python/generated_cases.c.h" + #line 5039 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index ac3d800d8fe0fe..82c98235892287 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -1087,7 +1087,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { [JUMP_BACKWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, From b4efdf8cda8fbbd0ca53b457d5f6e46a59348caf Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 3 Jul 2023 21:29:44 +0100 Subject: [PATCH 31/31] GH-106330: Fix matching of empty path in `pathlib.PurePath.match()` (GH-106331) We match paths using the `_lines` attribute, which is derived from the path's string representation. The bug arises because an empty path's string representation is `'.'` (not `''`), which is matched by the `'*'` wildcard. --- Lib/pathlib.py | 8 ++++++-- Lib/test/test_pathlib.py | 4 ++++ .../2023-07-02-10-56-41.gh-issue-106330.QSkIUH.rst | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-02-10-56-41.gh-issue-106330.QSkIUH.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index e15718dc98d677..f3813e04109904 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -463,8 +463,12 @@ def _lines(self): try: return self._lines_cached except AttributeError: - trans = _SWAP_SEP_AND_NEWLINE[self._flavour.sep] - self._lines_cached = str(self).translate(trans) + path_str = str(self) + if path_str == '.': + self._lines_cached = '' + else: + trans = _SWAP_SEP_AND_NEWLINE[self._flavour.sep] + self._lines_cached = path_str.translate(trans) return self._lines_cached def __eq__(self, other): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 464a835212d472..eb2b0cfb26e85f 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -384,6 +384,10 @@ def test_match_common(self): self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P().match('*')) + self.assertTrue(P().match('**')) + self.assertFalse(P().match('**/*')) def test_ordering_common(self): # Ordering is tuple-alike. diff --git a/Misc/NEWS.d/next/Library/2023-07-02-10-56-41.gh-issue-106330.QSkIUH.rst b/Misc/NEWS.d/next/Library/2023-07-02-10-56-41.gh-issue-106330.QSkIUH.rst new file mode 100644 index 00000000000000..c1f55ab658b517 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-02-10-56-41.gh-issue-106330.QSkIUH.rst @@ -0,0 +1,2 @@ +Fix incorrect matching of empty paths in :meth:`pathlib.PurePath.match`. +This bug was introduced in Python 3.12.0 beta 1.