From 7a3518e43aa50ea57fd35863da831052749b6115 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 26 Feb 2024 09:22:09 +0000 Subject: [PATCH 01/82] gh-115881: Ensure `ast.parse()` parses conditional context managers even with low `feature_version` passed (#115920) --- Grammar/python.gram | 2 +- Lib/test/test_ast.py | 12 ++++-------- Lib/test/test_type_comments.py | 2 +- .../2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst | 4 ++++ Parser/parser.c | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 174b4dbb6f7842..797c195a0a91ba 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -393,7 +393,7 @@ for_stmt[stmt_ty]: with_stmt[stmt_ty]: | invalid_with_stmt_indent | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' tc=[TYPE_COMMENT] b=block { - CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3789ac22e3899c..d49c149da42592 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1045,19 +1045,15 @@ def test_positional_only_feature_version(self): with self.assertRaises(SyntaxError): ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - def test_assignment_expression_feature_version(self): ast.parse('(x := 0)', feature_version=(3, 8)) with self.assertRaises(SyntaxError): ast.parse('(x := 0)', feature_version=(3, 7)) + def test_conditional_context_managers_parse_with_low_feature_version(self): + # regression test for gh-115881 + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) + def test_exception_groups_feature_version(self): code = dedent(''' try: ... diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 5a911da56f8f8a..ee8939f62d082c 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -309,7 +309,7 @@ def test_withstmt(self): self.assertEqual(tree.body[0].type_comment, None) def test_parenthesized_withstmt(self): - for tree in self.parse_all(parenthesized_withstmt, minver=9): + for tree in self.parse_all(parenthesized_withstmt): self.assertEqual(tree.body[0].type_comment, "int") self.assertEqual(tree.body[1].type_comment, "int") tree = self.classic_parse(parenthesized_withstmt) diff --git a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst new file mode 100644 index 00000000000000..99bccb265ff80c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst @@ -0,0 +1,4 @@ +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the +grammar made as part of gh-94949. diff --git a/Parser/parser.c b/Parser/parser.c index 779b18e9650e9f..f1170c26197452 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -6603,7 +6603,7 @@ with_stmt_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = CHECK_VERSION ( stmt_ty , 9 , "Parenthesized context managers are" , _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ) ); + _res = _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; From 915d7dd090387b52f62bdc2f572413bc87297cee Mon Sep 17 00:00:00 2001 From: Antti Haapala Date: Mon, 26 Feb 2024 11:22:54 +0200 Subject: [PATCH 02/82] gh-115091: Remove a left-over sentence that refers to Py_OptimizeFlag from ctypes documentation (GH-115092) Remove a left-over sentence that refers to Py_OptimizeFlag Remove a left-over sentence that refers to an example that was present in Python 3.10 and was using ``Py_OptimizeFlag``. --- Doc/library/ctypes.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 702955659a3bb9..eed18201e3ede0 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1117,10 +1117,6 @@ api:: >>> print(hex(version.value)) 0x30c00a0 -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - An extended example which also demonstrates the use of pointers accesses the :c:data:`PyImport_FrozenModules` pointer exported by Python. From 37f5d06b1bf830048c09ed967bb2cda945d56541 Mon Sep 17 00:00:00 2001 From: Phil Elson Date: Mon, 26 Feb 2024 10:53:20 +0100 Subject: [PATCH 03/82] Doc: Clarify the return type of Event.wait when timeout is used (GH-104168) --- Doc/library/threading.rst | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 5fbf9379b8202c..cb511471579b69 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -987,18 +987,15 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. method:: wait(timeout=None) - Block until the internal flag is true. If the internal flag is true on - entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. + Block as long as the internal flag is false and the timeout, if given, + has not expired. The return value represents the + reason that this blocking method returned; ``True`` if returning because + the internal flag is set to true, or ``False`` if a timeout is given and + the the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - This method returns ``True`` if and only if the internal flag has been set to - true, either before the wait call or after the wait starts, so it will - always return ``True`` except if a timeout is given and the operation - times out. + floating point number specifying a timeout for the operation in seconds, + or fractions thereof. .. versionchanged:: 3.1 Previously, the method always returned ``None``. From b7383b8b71d49c761480ae9a8b2111644310e61d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 26 Feb 2024 15:32:27 +0300 Subject: [PATCH 04/82] gh-115931: Fix `SyntaxWarning`s in `test_unparse` (#115935) --- Lib/test/test_unparse.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 77ce18cbf4cbfb..106704ba8c9c2d 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -650,9 +650,18 @@ def test_multiquote_joined_string(self): self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) def test_backslash_in_format_spec(self): - self.check_ast_roundtrip("""f"{x:\\ }" """) + import re + msg = re.escape("invalid escape sequence '\\ '") + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + self.check_ast_roundtrip("""f"{x:\\\\ }" """) - self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) def test_quote_in_format_spec(self): From 015b97d19a24a169cc3c0939119e1228791e4253 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 26 Feb 2024 13:57:09 +0100 Subject: [PATCH 05/82] gh-115823: Calculate correctly error locations when dealing with implicit encodings (#115824) --- Lib/test/test_exceptions.py | 1 + ...-02-22-16-17-53.gh-issue-115823.c1TreJ.rst | 3 +++ Parser/pegen_errors.c | 20 +++++++++---------- 3 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c7e76414ff0715..c5eff8ad8ccca1 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -301,6 +301,7 @@ def baz(): { 6 0="""''', 5, 13) + check('b"fooжжж"'.encode(), 1, 1, 1, 10) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 7) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst new file mode 100644 index 00000000000000..8cda4c9343d4d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst @@ -0,0 +1,3 @@ +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e15673d02dd3b0..e8f11a67e50fa0 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -369,20 +369,18 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t col_number = col_offset; Py_ssize_t end_col_number = end_col_offset; - if (p->tok->encoding != NULL) { - col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); - if (col_number < 0) { + col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); + if (col_number < 0) { + goto error; + } + + if (end_col_offset > 0) { + end_col_number = _PyPegen_byte_offset_to_character_offset(error_line, end_col_offset); + if (end_col_number < 0) { goto error; } - if (end_col_number > 0) { - Py_ssize_t end_col_offset = _PyPegen_byte_offset_to_character_offset(error_line, end_col_number); - if (end_col_offset < 0) { - goto error; - } else { - end_col_number = end_col_offset; - } - } } + tmp = Py_BuildValue("(OnnNnn)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number); if (!tmp) { goto error; From 07824995a056b2894adac69813444307835cc245 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 26 Feb 2024 10:18:30 -0500 Subject: [PATCH 06/82] gh-113706: Update comment about long int representation (#113707) --- Include/cpython/longintrepr.h | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f5ccbb704e8bb8..f037c7bb90deda 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -62,21 +62,32 @@ typedef long stwodigits; /* signed variant of twodigits */ #define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. + + Long integers are made up of a number of 30- or 15-bit digits, depending on + the platform. The number of digits (ndigits) is stored in the high bits of + the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS). + The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. - In a normalized number, ob_digit[abs(ob_size)-1] (the most significant + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + + The sign of the value is stored in the lower 2 bits of lv_tag. + + - 0: Positive + - 1: Zero + - 2: Negative + + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. + + In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= PyLong_MASK. + The allocation function takes care of allocating extra memory - so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. + so that ob_digit[0] ... ob_digit[ndigits-1] are actually available. We always allocate memory for at least one digit, so accessing ob_digit[0] - is always safe. However, in the case ob_size == 0, the contents of + is always safe. However, in the case ndigits == 0, the contents of ob_digit[0] may be undefined. - - CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. */ typedef struct _PyLongValue { From 5a832922130908994d313b56a3345ff410a0e11a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 26 Feb 2024 11:11:38 -0500 Subject: [PATCH 07/82] Add Jason as an owner of configparser to coordinate backport concerns. (#115885) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5dbfbbb8ebaf7e..7e20294e90e50d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -249,3 +249,7 @@ Lib/test/test_interpreters/ @ericsnowcurrently # SBOM /Misc/sbom.spdx.json @sethmlarson /Tools/build/generate_sbom.py @sethmlarson + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco From 7259480957e10359cc5ab8786f32f197c88e274c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 26 Feb 2024 08:32:44 -0800 Subject: [PATCH 08/82] GH-115802: JIT "small" code for macOS and Linux (GH-115826) --- Python/jit.c | 119 ++++++++++++++++++++++++++++++++++++------ Tools/jit/_schema.py | 10 ++++ Tools/jit/_targets.py | 68 +++++++++++++++++++----- 3 files changed, 168 insertions(+), 29 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 839414bd810677..ac2c60ed925a26 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -47,18 +47,18 @@ jit_error(const char *message) PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); } -static char * +static unsigned char * jit_alloc(size_t size) { assert(size); assert(size % get_page_size() == 0); #ifdef MS_WINDOWS int flags = MEM_COMMIT | MEM_RESERVE; - char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); + unsigned char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); int failed = memory == NULL; #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; - char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + unsigned char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); int failed = memory == MAP_FAILED; #endif if (failed) { @@ -69,7 +69,7 @@ jit_alloc(size_t size) } static int -jit_free(char *memory, size_t size) +jit_free(unsigned char *memory, size_t size) { assert(size); assert(size % get_page_size() == 0); @@ -86,7 +86,7 @@ jit_free(char *memory, size_t size) } static int -mark_executable(char *memory, size_t size) +mark_executable(unsigned char *memory, size_t size) { if (size == 0) { return 0; @@ -113,7 +113,7 @@ mark_executable(char *memory, size_t size) } static int -mark_readable(char *memory, size_t size) +mark_readable(unsigned char *memory, size_t size) { if (size == 0) { return 0; @@ -169,18 +169,20 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, // Fill all of stencil's holes in the memory pointed to by base, using the // values in patches. static void -patch(char *base, const Stencil *stencil, uint64_t *patches) +patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) { for (uint64_t i = 0; i < stencil->holes_size; i++) { const Hole *hole = &stencil->holes[i]; - void *location = base + hole->offset; + unsigned char *location = base + hole->offset; uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; + uint8_t *loc8 = (uint8_t *)location; uint32_t *loc32 = (uint32_t *)location; uint64_t *loc64 = (uint64_t *)location; // LLD is a great reference for performing relocations... just keep in // mind that Tools/jit/build.py does filtering and preprocessing for us! // Here's a good place to start for each platform: // - aarch64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h // - aarch64-unknown-linux-gnu: @@ -208,6 +210,47 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) // 64-bit absolute address. *loc64 = value; continue; + case HoleKind_R_X86_64_GOTPCRELX: + case HoleKind_R_X86_64_REX_GOTPCRELX: + case HoleKind_X86_64_RELOC_GOT: + case HoleKind_X86_64_RELOC_GOT_LOAD: { + // 32-bit relative address. + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; + } + } + } + // Fall through... + case HoleKind_R_X86_64_GOTPCREL: + case HoleKind_R_X86_64_PC32: + case HoleKind_X86_64_RELOC_SIGNED: + case HoleKind_X86_64_RELOC_BRANCH: + // 32-bit relative address. + value -= (uint64_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + loc32[0] = (uint32_t)value; + continue; case HoleKind_R_AARCH64_CALL26: case HoleKind_R_AARCH64_JUMP26: // 28-bit relative branch. @@ -249,10 +292,53 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) set_bits(loc32, 5, value, 48, 16); continue; case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: + case HoleKind_R_AARCH64_ADR_GOT_PAGE: // 21-bit count of pages between this page and an absolute address's // page... I know, I know, it's weird. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). assert(IS_AARCH64_ADRP(*loc32)); + // Try to relax the pair of GOT loads into an immediate value: + const Hole *next_hole = &stencil->holes[i + 1]; + if (i + 1 < stencil->holes_size && + (next_hole->kind == HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12 || + next_hole->kind == HoleKind_R_AARCH64_LD64_GOT_LO12_NC) && + next_hole->offset == hole->offset + 4 && + next_hole->symbol == hole->symbol && + next_hole->addend == hole->addend && + next_hole->value == hole->value) + { + unsigned char rd = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + unsigned char rt = get_bits(loc32[1], 0, 5); + unsigned char rn = get_bits(loc32[1], 5, 5); + assert(rd == rn && rn == rt); + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd; + loc32[1] = 0xD503201F; + i++; + continue; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | rd; + i++; + continue; + } + relaxed = (uint64_t)value - (uint64_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr x0, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | rd; + loc32[1] = 0xD503201F; + i++; + continue; + } + } // Number of pages between this page and the value's page: value = (value >> 12) - ((uint64_t)location >> 12); // Check that we're not out of range of 21 signed bits: @@ -264,6 +350,7 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) set_bits(loc32, 5, value, 2, 19); continue; case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: + case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: // 12-bit low part of an absolute address. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGE21 (above). assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); @@ -285,7 +372,7 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) } static void -copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches) +copy_and_patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) { memcpy(base, stencil->body, stencil->body_size); patch(base, stencil, patches); @@ -294,8 +381,8 @@ copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches) static void emit(const StencilGroup *group, uint64_t patches[]) { - copy_and_patch((char *)patches[HoleValue_CODE], &group->code, patches); - copy_and_patch((char *)patches[HoleValue_DATA], &group->data, patches); + copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); + copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); } // Compiles executor in-place. Don't forget to call _PyJIT_Free later! @@ -316,14 +403,14 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size assert((page_size & (page_size - 1)) == 0); code_size += page_size - (code_size & (page_size - 1)); data_size += page_size - (data_size & (page_size - 1)); - char *memory = jit_alloc(code_size + data_size); + unsigned char *memory = jit_alloc(code_size + data_size); if (memory == NULL) { return -1; } // Loop again to emit the code: - char *code = memory; - char *data = memory + code_size; - char *top = code; + unsigned char *code = memory; + unsigned char *data = memory + code_size; + unsigned char *top = code; if (trace[0].opcode == _START_EXECUTOR) { // Don't want to execute this more than once: top += stencil_groups[_START_EXECUTOR].code.body_size; @@ -360,7 +447,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size void _PyJIT_Free(_PyExecutorObject *executor) { - char *memory = (char *)executor->jit_code; + unsigned char *memory = (unsigned char *)executor->jit_code; size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 8eeb78e6cd69ee..975ca650a13c1a 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -8,13 +8,23 @@ "IMAGE_REL_AMD64_ADDR64", "IMAGE_REL_I386_DIR32", "R_AARCH64_ABS64", + "R_AARCH64_ADR_GOT_PAGE", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", "R_AARCH64_MOVW_UABS_G2_NC", "R_AARCH64_MOVW_UABS_G3", "R_X86_64_64", + "R_X86_64_GOTPCREL", + "R_X86_64_GOTPCRELX", + "R_X86_64_PC32", + "R_X86_64_REX_GOTPCRELX", + "X86_64_RELOC_BRANCH", + "X86_64_RELOC_GOT", + "X86_64_RELOC_GOT_LOAD", + "X86_64_RELOC_SIGNED", "X86_64_RELOC_UNSIGNED", ] diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 6c1d440324c505..06dc4e7acc6c91 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -37,6 +37,7 @@ class _Target(typing.Generic[_S, _R]): triple: str _: dataclasses.KW_ONLY alignment: int = 1 + args: typing.Sequence[str] = () prefix: str = "" debug: bool = False force: bool = False @@ -121,21 +122,14 @@ async def _compile( "-fno-builtin", # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: "-fno-jump-tables", - # Position-independent code adds indirection to every load and jump: - "-fno-pic", + "-fno-plt", # Don't make calls to weird stack-smashing canaries: "-fno-stack-protector", - # We have three options for code model: - # - "small": the default, assumes that code and data reside in the - # lowest 2GB of memory (128MB on aarch64) - # - "medium": assumes that code resides in the lowest 2GB of memory, - # and makes no assumptions about data (not available on aarch64) - # - "large": makes no assumptions about either code or data - "-mcmodel=large", "-o", f"{o}", "-std=c11", f"{c}", + *self.args, ] await _llvm.run("clang", args, echo=self.verbose) return await self._parse(o) @@ -284,7 +278,23 @@ def _handle_section( def _handle_relocation( self, base: int, relocation: _schema.ELFRelocation, raw: bytes ) -> _stencils.Hole: + symbol: str | None match relocation: + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "R_AARCH64_ADR_GOT_PAGE" + | "R_AARCH64_LD64_GOT_LO12_NC" + | "R_X86_64_GOTPCREL" + | "R_X86_64_GOTPCRELX" + | "R_X86_64_REX_GOTPCRELX" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s case { "Addend": addend, "Offset": offset, @@ -358,6 +368,34 @@ def _handle_relocation( s = s.removeprefix(self.prefix) value, symbol = _stencils.HoleValue.GOT, s addend = 0 + case { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Value": s}, + "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + } | { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) case { "Offset": offset, "Section": {"Value": s}, @@ -379,15 +417,19 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" if re.fullmatch(r"aarch64-apple-darwin.*", host): - return _MachO(host, alignment=8, prefix="_") + args = ["-mcmodel=large"] + return _MachO(host, alignment=8, args=args, prefix="_") if re.fullmatch(r"aarch64-.*-linux-gnu", host): - return _ELF(host, alignment=8) + args = ["-mcmodel=large"] + return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): - return _COFF(host, prefix="_") + args = ["-mcmodel=large"] + return _COFF(host, args=args, prefix="_") if re.fullmatch(r"x86_64-apple-darwin.*", host): return _MachO(host, prefix="_") if re.fullmatch(r"x86_64-pc-windows-msvc", host): - return _COFF(host) + args = ["-mcmodel=large"] + return _COFF(host, args=args) if re.fullmatch(r"x86_64-.*-linux-gnu", host): return _ELF(host) raise ValueError(host) From c0fdfba7ff981c55ac13325e4dddaf382601b246 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 26 Feb 2024 08:42:53 -0800 Subject: [PATCH 09/82] Rename tier 2 redundancy eliminator to optimizer (#115888) The original name is just too much of a mouthful. --- .gitattributes | 2 +- .github/CODEOWNERS | 2 +- .github/workflows/jit.yml | 4 ++-- Lib/test/test_generated_cases.py | 4 ++-- Makefile.pre.in | 10 +++++----- PCbuild/regen.targets | 6 +++--- Python/optimizer_analysis.c | 8 ++++---- ...cy_eliminator_bytecodes.c => optimizer_bytecodes.c} | 0 ...ndancy_eliminator_cases.c.h => optimizer_cases.c.h} | 4 ++-- Tools/c-analyzer/cpython/_parser.py | 4 ++-- Tools/cases_generator/README.md | 6 +++--- ...r2_abstract_generator.py => optimizer_generator.py} | 10 +++++----- 12 files changed, 30 insertions(+), 30 deletions(-) rename Python/{tier2_redundancy_eliminator_bytecodes.c => optimizer_bytecodes.c} (100%) rename Python/{tier2_redundancy_eliminator_cases.c.h => optimizer_cases.c.h} (99%) rename Tools/cases_generator/{tier2_abstract_generator.py => optimizer_generator.py} (94%) diff --git a/.gitattributes b/.gitattributes index 159cd83ff7407d..4f82cea7480cfe 100644 --- a/.gitattributes +++ b/.gitattributes @@ -95,7 +95,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/tier2_redundancy_eliminator_cases.c.h generated +Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7e20294e90e50d..1d0bce111320ff 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,7 +38,7 @@ Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum Python/optimizer_analysis.c @Fidget-Spinner -Python/tier2_redundancy_eliminator_bytecodes.c @Fidget-Spinner +Python/optimizer_bytecodes.c @Fidget-Spinner Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69c7b45376a411..21d4603b8679ea 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,13 +5,13 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/tier2_redundancy_eliminator_bytecodes.c' + - 'Python/optimizer_bytecodes.c' push: paths: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/tier2_redundancy_eliminator_bytecodes.c' + - 'Python/optimizer_bytecodes.c' workflow_dispatch: concurrency: diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 0d2ccd558d436d..6a5586822c2491 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -33,7 +33,7 @@ def skip_if_different_mount_drives(): import parser from stack import Stack import tier1_generator - import tier2_abstract_generator + import optimizer_generator def handle_stderr(): @@ -841,7 +841,7 @@ def run_cases_test(self, input: str, input2: str, expected: str): temp_input.flush() with handle_stderr(): - tier2_abstract_generator.generate_tier2_abstract_from_files( + optimizer_generator.generate_tier2_abstract_from_files( [self.temp_input_filename, self.temp_input2_filename], self.temp_output_filename ) diff --git a/Makefile.pre.in b/Makefile.pre.in index 8893da816e9336..e4c64aea80d22c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1889,9 +1889,9 @@ regen-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_abstract_generator.py \ - -o $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h.new \ - $(srcdir)/Python/tier2_redundancy_eliminator_bytecodes.c \ + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + -o $(srcdir)/Python/optimizer_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c @@ -1904,7 +1904,7 @@ regen-cases: $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1927,7 +1927,7 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h + $(srcdir)/Python/optimizer_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 8f31803dbb752a..f36387231a0d3a 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -31,8 +31,8 @@ <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> - <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\tier2_redundancy_eliminator_bytecodes.c;"/> - <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\tier2_redundancy_eliminator_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> + <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> @@ -98,7 +98,7 @@ WorkingDirectory="$(PySourcePath)" /> - diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 47bfc8cf1061d9..1a9e82903ffbc0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,5 +1,5 @@ /* - * This file contains the support code for CPython's uops redundancy eliminator. + * This file contains the support code for CPython's uops optimizer. * It also performs some simple optimizations. * It performs a traditional data-flow analysis[1] over the trace of uops. * Using the information gained, it chooses to emit, or skip certain instructions @@ -606,7 +606,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, /* 1 for success, 0 for not ready, cannot error at the moment. */ static int -uop_redundancy_eliminator( +optimize_uops( PyCodeObject *co, _PyUOpInstruction *trace, int trace_len, @@ -638,7 +638,7 @@ uop_redundancy_eliminator( _PyUOpName(opcode), oparg); switch (opcode) { -#include "tier2_redundancy_eliminator_cases.c.h" +#include "optimizer_cases.c.h" default: DPRINTF(1, "Unknown opcode in abstract interpreter\n"); @@ -812,7 +812,7 @@ _Py_uop_analyze_and_optimize( char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); if (uop_optimize != NULL && *uop_optimize > '0') { - err = uop_redundancy_eliminator( + err = optimize_uops( (PyCodeObject *)frame->f_executable, buffer, buffer_size, curr_stacklen, dependencies); } diff --git a/Python/tier2_redundancy_eliminator_bytecodes.c b/Python/optimizer_bytecodes.c similarity index 100% rename from Python/tier2_redundancy_eliminator_bytecodes.c rename to Python/optimizer_bytecodes.c diff --git a/Python/tier2_redundancy_eliminator_cases.c.h b/Python/optimizer_cases.c.h similarity index 99% rename from Python/tier2_redundancy_eliminator_cases.c.h rename to Python/optimizer_cases.c.h index ca341e4dde5d93..e8146dccbdfcd1 100644 --- a/Python/tier2_redundancy_eliminator_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1,6 +1,6 @@ -// This file is generated by Tools/cases_generator/tier2_abstract_generator.py +// This file is generated by Tools/cases_generator/optimizer_generator.py // from: -// Python/tier2_redundancy_eliminator_bytecodes.c +// Python/optimizer_bytecodes.c // Do not edit! case _NOP: { diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 8482bf849aaacd..12010f0e9c0549 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -83,11 +83,11 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h -Python/tier2_redundancy_eliminator_cases.c.h +Python/optimizer_cases.c.h # not actually source Python/bytecodes.c -Python/tier2_redundancy_eliminator_bytecodes.c +Python/optimizer_bytecodes.c # mimalloc Objects/mimalloc/*.c diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index d35a868b42ea9e..fb512c4646b851 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -13,9 +13,9 @@ What's currently here: - `parser.py` helper for interactions with `parsing.py` - `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `tier2_abstract_generator.py`: reads `Python/bytecodes.c` and - `Python/tier2_redundancy_eliminator_bytecodes.c` and writes - `Python/tier2_redundancy_eliminator_cases.c.h` +- `optimizer_generator.py`: reads `Python/bytecodes.c` and + `Python/optimizer_bytecodes.c` and writes + `Python/optimizer_cases.c.h` - `stack.py`: code to handle generalized stack effects - `cwriter.py`: code which understands tokens and how to format C code; main class: `CWriter` diff --git a/Tools/cases_generator/tier2_abstract_generator.py b/Tools/cases_generator/optimizer_generator.py similarity index 94% rename from Tools/cases_generator/tier2_abstract_generator.py rename to Tools/cases_generator/optimizer_generator.py index 58c3110dc3facb..aa3f4ecb11d58b 100644 --- a/Tools/cases_generator/tier2_abstract_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -1,6 +1,6 @@ -"""Generate the cases for the tier 2 redundancy eliminator/abstract interpreter. -Reads the instruction definitions from bytecodes.c. and tier2_redundancy_eliminator.bytecodes.c -Writes the cases to tier2_redundancy_eliminator_cases.c.h, which is #included in Python/optimizer_analysis.c. +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. """ import argparse @@ -30,8 +30,8 @@ from lexer import Token from stack import StackOffset, Stack, SizeMismatch, UNUSED -DEFAULT_OUTPUT = ROOT / "Python/tier2_redundancy_eliminator_cases.c.h" -DEFAULT_ABSTRACT_INPUT = ROOT / "Python/tier2_redundancy_eliminator_bytecodes.c" +DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_ABSTRACT_INPUT = ROOT / "Python/optimizer_bytecodes.c" def validate_uop(override: Uop, uop: Uop) -> None: From 96c10c648565c7406d5606099dbbb937310c26dc Mon Sep 17 00:00:00 2001 From: Yuriy Chernyshov Date: Mon, 26 Feb 2024 18:21:55 +0100 Subject: [PATCH 10/82] gh-115882: Reference Unknwn.h for ctypes on Windows (GH-115350) This allows the module to be compiled with WIN32_LEAN_AND_MEAN enabled --- .../next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst | 1 + Modules/_ctypes/ctypes.h | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst diff --git a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst new file mode 100644 index 00000000000000..5492a804255838 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst @@ -0,0 +1 @@ +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1989723f6f3dbb..02f48a9ed55843 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -32,6 +32,10 @@ #endif #endif +#ifdef MS_WIN32 +#include // for IUnknown interface +#endif + typedef struct { PyTypeObject *DictRemover_Type; PyTypeObject *PyCArg_Type; From b05afdd5ec325bdb4cc89bb3be177ed577bea41f Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 26 Feb 2024 12:51:47 -0500 Subject: [PATCH 11/82] gh-115168: Add pystats counter for invalidated executors (GH-115169) --- Include/cpython/optimizer.h | 4 ++-- Include/cpython/pystats.h | 1 + Modules/_testinternalcapi.c | 2 +- Python/instrumentation.c | 6 +++--- Python/optimizer.c | 10 ++++++++-- Python/optimizer_analysis.c | 2 +- Python/pylifecycle.c | 4 ++-- Python/pystate.c | 2 +- Python/specialize.c | 1 + Python/sysmodule.c | 2 +- Tools/scripts/summarize_stats.py | 11 ++++++++++- 11 files changed, 31 insertions(+), 14 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index fe54d1ddfe6129..8fc9fb62aebdb4 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -100,8 +100,8 @@ void _Py_ExecutorClear(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index db9aaedec950e4..887fbbedf88502 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -115,6 +115,7 @@ typedef struct _optimization_stats { uint64_t inner_loop; uint64_t recursive_call; uint64_t low_confidence; + uint64_t executors_invalidated; UOpStats opcode[512]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 0d23b1899f22e4..5b714ca3f3dc33 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1035,7 +1035,7 @@ static PyObject * invalidate_executors(PyObject *self, PyObject *obj) { PyInterpreterState *interp = PyInterpreterState_Get(); - _Py_Executors_InvalidateDependency(interp, obj); + _Py_Executors_InvalidateDependency(interp, obj, 1); Py_RETURN_NONE; } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 878d19f0552bf5..6f1bc2e0a107df 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1599,7 +1599,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } - _Py_Executors_InvalidateDependency(interp, code); + _Py_Executors_InvalidateDependency(interp, code, 1); int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation * data for potential statically allocated code @@ -1820,7 +1820,7 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) return -1; } set_global_version(tstate, new_version); - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 1); return instrument_all_executing_code_objects(interp); } @@ -1850,7 +1850,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent /* Force instrumentation update */ code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } - _Py_Executors_InvalidateDependency(interp, code); + _Py_Executors_InvalidateDependency(interp, code, 1); if (_Py_Instrument(code, interp)) { return -1; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 6b2ba3addefc95..c04ee17ee2171d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1348,7 +1348,7 @@ _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) * May cause other executors to be invalidated as well */ void -_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) +_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation) { _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); @@ -1360,6 +1360,9 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { _Py_ExecutorClear(exec); + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } } exec = next; } @@ -1367,7 +1370,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) /* Invalidate all executors */ void -_Py_Executors_InvalidateAll(PyInterpreterState *interp) +_Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) { while (interp->executor_list_head) { _PyExecutorObject *executor = interp->executor_list_head; @@ -1378,5 +1381,8 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) else { _Py_ExecutorClear(executor); } + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } } } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1a9e82903ffbc0..e9751291e2fd5a 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -405,7 +405,7 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, { RARE_EVENT_STAT_INC(watched_globals_modification); assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); - _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict, 1); increment_mutations(dict); PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); return 0; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 04487345f7ec05..3a2c0a450ac9d9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -612,7 +612,7 @@ builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, Py { PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 1); } RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; @@ -1628,7 +1628,7 @@ finalize_modules(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; // Invalidate all executors and turn off tier 2 optimizer - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 0); _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); Py_XDECREF(old); diff --git a/Python/pystate.c b/Python/pystate.c index bb8e24c1dbe12f..a80c1b7fb9c866 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2666,7 +2666,7 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, return; } if (eval_frame != NULL) { - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 1); } RARE_EVENT_INC(set_eval_frame_func); interp->eval_frame = eval_frame; diff --git a/Python/specialize.c b/Python/specialize.c index 871979d92298b6..f83d8a9ceb0282 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -236,6 +236,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 69b6d886ccc3e9..1bfd031fdd26b2 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2138,7 +2138,7 @@ sys__clear_internal_caches_impl(PyObject *module) /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 0); PyType_ClearCache(); Py_RETURN_NONE; } diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 6b60b59b3b0e79..2925e096f4d95e 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -451,6 +451,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: inner_loop = self._data["Optimization inner loop"] recursive_call = self._data["Optimization recursive call"] low_confidence = self._data["Optimization low confidence"] + executors_invalidated = self._data["Executors invalidated"] return { Doc( @@ -493,11 +494,19 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: "A trace is abandoned because the likelihood of the jump to top being taken " "is too low.", ): (low_confidence, attempts), + Doc( + "Executors invalidated", + "The number of executors that were invalidated due to watched " + "dictionary changes.", + ): (executors_invalidated, created), Doc("Traces executed", "The number of traces that were executed"): ( executed, None, ), - Doc("Uops executed", "The total number of uops (micro-operations) that were executed"): ( + Doc( + "Uops executed", + "The total number of uops (micro-operations) that were executed", + ): ( uops, executed, ), From 68c79d21fa791d7418a858b7aa4604880e988a02 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 26 Feb 2024 20:07:41 +0200 Subject: [PATCH 12/82] gh-112006: Fix inspect.unwrap() for types where __wrapped__ is a data descriptor (GH-115540) This also fixes inspect.Signature.from_callable() for builtins classmethod() and staticmethod(). --- Lib/inspect.py | 10 ++---- Lib/test/test_inspect/test_inspect.py | 32 +++++++++++++++---- ...-02-15-23-42-54.gh-issue-112006.4wxcK-.rst | 3 ++ 3 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index da504037ac282c..9191d4761b2c76 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -762,18 +762,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c5a6de5993fad4..9dc37852ec205a 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3137,6 +3137,10 @@ def m1d(*args, **kwargs): int)) def test_signature_on_classmethod(self): + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @classmethod def foo(cls, arg1, *, arg2=1): @@ -3155,6 +3159,10 @@ def foo(cls, arg1, *, arg2=1): ...)) def test_signature_on_staticmethod(self): + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @staticmethod def foo(cls, *, arg): @@ -3678,16 +3686,20 @@ class Bar(Spam, Foo): ((('a', ..., ..., "positional_or_keyword"),), ...)) - class Wrapped: - pass - Wrapped.__wrapped__ = lambda a: None - self.assertEqual(self.signature(Wrapped), + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), ((('a', ..., ..., "positional_or_keyword"),), ...)) # wrapper loop: - Wrapped.__wrapped__ = Wrapped + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper with self.assertRaisesRegex(ValueError, 'wrapper loop'): - self.signature(Wrapped) + self.signature(wrapper) def test_signature_on_lambdas(self): self.assertEqual(self.signature((lambda a=10: a)), @@ -4999,6 +5011,14 @@ def test_recursion_limit(self): with self.assertRaisesRegex(ValueError, 'wrapper loop'): inspect.unwrap(obj) + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') diff --git a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst new file mode 100644 index 00000000000000..32af2bd24e54f2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst @@ -0,0 +1,3 @@ +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. From 72cff8d8e5a476d3406efb0491452bf9d6b02feb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 26 Feb 2024 20:29:49 +0200 Subject: [PATCH 13/82] gh-113942: Show functions implemented as builtin methods (GH-115306) Pydoc no longer skips global functions implemented as builtin methods, such as MethodDescriptorType and WrapperDescriptorType. --- Lib/pydoc.py | 12 ++++++------ Lib/test/test_pydoc/pydocfodder.py | 4 ++++ Lib/test/test_pydoc/test_pydoc.py | 12 ++++++++++++ .../2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst | 2 ++ 4 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d32fa8d0504417..b0193b4a85164a 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -855,9 +855,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -1299,9 +1299,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] diff --git a/Lib/test/test_pydoc/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py index 27037e048db819..3cc2d5bd57fe5b 100644 --- a/Lib/test/test_pydoc/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -81,6 +81,8 @@ def B_classmethod(cls, x): A_method_ref = A().A_method A_method_alias = A.A_method B_method_alias = B_method + count = list.count # same name + list_count = list.count __repr__ = object.__repr__ # same name object_repr = object.__repr__ get = {}.get # same name @@ -180,5 +182,7 @@ def __call__(self, inst): B_method2 = B.B_method count = list.count # same name list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ get = {}.get # same name dict_get = {}.get diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index b07d9119e49401..9d40234ed01697 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1686,6 +1686,8 @@ def test_text_doc_routines_in_class(self, cls=pydocfodder.B): self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) self.assertIn(' | global_func_alias = global_func(x, y)', lines) self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) self.assertIn(' | __repr__(self, /) from builtins.object', lines) self.assertIn(' | object_repr = __repr__(self, /)', lines) @@ -1714,6 +1716,8 @@ def test_html_doc_routines_in_class(self, cls=pydocfodder.B): self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) self.assertIn('global_func_alias = global_func(x, y)', lines) self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) self.assertIn('__repr__(self, /) from builtins.object', lines) self.assertIn('object_repr = __repr__(self, /)', lines) @@ -1757,6 +1761,10 @@ def test_text_doc_routines_in_module(self): # unbound methods self.assertIn(' B_method(self)', lines) self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) def test_html_doc_routines_in_module(self): doc = pydoc.HTMLDoc() @@ -1782,6 +1790,10 @@ def test_html_doc_routines_in_module(self): # unbound methods self.assertIn(' B_method(self)', lines) self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) @unittest.skipIf( diff --git a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst new file mode 100644 index 00000000000000..2da43a43b78798 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst @@ -0,0 +1,2 @@ +:mod:`pydoc` no longer skips global functions implemented as builtin methods, +such as :class:`~type.MethodDescriptorType` and :class:`~type.WrapperDescriptorType`. From 37a13b941394a1337576c67bff35d4a44a1157e2 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 26 Feb 2024 19:14:14 +0000 Subject: [PATCH 14/82] gh-115582: Make default PC/pyconfig.h work for free-threaded builds with manual /DPy_GIL_DISABLED (GH-115850) --- .../Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst | 3 +++ PC/pyconfig.h.in | 7 ++++++- PCbuild/pythoncore.vcxproj | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst diff --git a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst new file mode 100644 index 00000000000000..f2e82bf6a3e028 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst @@ -0,0 +1,3 @@ +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. This +is expected to change in future releases. diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index 8bbf877a5bb5ed..d72d6282c2806f 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -95,7 +95,12 @@ WIN32 is still required for the locale module. #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Define to 1 if you want to disable the GIL */ -#undef Py_GIL_DISABLED +/* Uncomment the definition for free-threaded builds, or define it manually + * when compiling extension modules. Note that we test with #ifdef, so + * defining as 0 will still disable the GIL. */ +#ifndef Py_GIL_DISABLED +/* #define Py_GIL_DISABLED 1 */ +#endif /* Compiler specific defines */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c7b698f0e17a39..ef21f85107bc32 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -677,7 +677,7 @@ $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) Date: Mon, 26 Feb 2024 16:20:39 -0300 Subject: [PATCH 15/82] gh-77956: Add the words 'default' and 'version' help text localizable (GH-12711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: paul.j3 Co-authored-by: Jérémie Detrey --- Lib/argparse.py | 6 ++++-- .../next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index f86658baf7f2ba..c16f538ae1535a 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -708,7 +708,7 @@ def _get_help_string(self, action): if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' + help += _(' (default: %(default)s)') return help @@ -1159,8 +1159,10 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit", + help=None, deprecated=False): + if help is None: + help = _("show program's version number and exit") super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, diff --git a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst new file mode 100644 index 00000000000000..2a663ac7940dcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst @@ -0,0 +1 @@ +Add 'default' and 'version' help text for localization in argparse. From de2a73dc4649b110351fce789de0abb14c460b97 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 27 Feb 2024 03:04:44 +0700 Subject: [PATCH 16/82] bpo-45101: Add consistency in usage message IO between 2 versions of python-config (GH-28162) On --help output to stdout. On error output to stderr. --- .../Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst | 1 + Misc/python-config.in | 3 ++- Misc/python-config.sh.in | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst new file mode 100644 index 00000000000000..48a09da7822915 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst @@ -0,0 +1 @@ +Add consistency in usage message IO between 2 versions of python-config. diff --git a/Misc/python-config.in b/Misc/python-config.in index 81c3316e334a48..dd5d161ab2286f 100644 --- a/Misc/python-config.in +++ b/Misc/python-config.in @@ -13,7 +13,8 @@ valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', def exit_with_usage(code=1): print("Usage: {0} [{1}]".format( - sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), file=sys.stderr) + sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), + file=sys.stdout if code == 0 else sys.stderr) sys.exit(code) try: diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..eb02223ddcd2c3 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,7 +4,12 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [[ "$1" -eq 0 ]]; then + echo "$USAGE" + else + echo "$USAGE" >&2 + fi exit $1 } From 6087315926fb185847a52559af063cc7d337d978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Detrey?= Date: Mon, 26 Feb 2024 23:05:01 +0100 Subject: [PATCH 17/82] bpo-44865: Fix yet one missing translations in argparse (GH-27668) --- Lib/argparse.py | 3 ++- .../next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index c16f538ae1535a..4200dd5e334ad5 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -223,7 +223,8 @@ def format_help(self): # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) + heading_text = _('%(heading)s:') % dict(heading=self.heading) + heading = '%*s%s\n' % (current_indent, '', heading_text) else: heading = '' diff --git a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst new file mode 100644 index 00000000000000..ecdb26cdd6edd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst @@ -0,0 +1 @@ +Add missing call to localization function in :mod:`argparse`. From af5f9d682c20c951b90e3c020eeccac386c9bbb0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 27 Feb 2024 09:51:17 +0100 Subject: [PATCH 18/82] gh-115720: Show number of leaks in huntrleaks progress reports (GH-115726) Instead of showing a dot for each iteration, show: - '.' for zero (on negative) leaks - number of leaks for 1-9 - 'X' if there are more leaks This allows more rapid iteration: when bisecting, I don't need to wait for the final report to see if the test still leaks. Also, show the full result if there are any non-zero entries. This shows negative entries, for the unfortunate cases where a reference is created and cleaned up in different runs. Test *failure* is still determined by the existing heuristic. --- Lib/test/libregrtest/refleak.py | 47 ++++++++++++++----- Lib/test/test_regrtest.py | 4 +- ...-02-20-15-47-41.gh-issue-115720.w8i8UG.rst | 2 + 3 files changed, 39 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 71a70af6882d16..f582c0d3e7ff13 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -88,9 +88,12 @@ def get_pooled_int(value): rc_before = alloc_before = fd_before = interned_before = 0 if not quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) results = None dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -116,13 +119,27 @@ def get_pooled_int(value): rc_after = gettotalrefcount() - interned_after * 2 fd_after = fd_count() - if not quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after @@ -158,14 +175,20 @@ def check_fd_deltas(deltas): ]: # ignore warmup runs deltas = deltas[warmups:] - if checker(deltas): + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(filename, "a", encoding="utf-8") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) return (failed, results) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index b80e0524593fc7..7e1eaa7d6a515e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1171,8 +1171,8 @@ def check_leak(self, code, what, *, run_workers=False): stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) diff --git a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst new file mode 100644 index 00000000000000..a03ee11d974251 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst @@ -0,0 +1,2 @@ +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. From 10fbcd6c5dc25bfe14e02fd93ef93498a393860c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 27 Feb 2024 10:51:26 +0000 Subject: [PATCH 19/82] GH-115816: Make tier2 optimizer symbols testable, and add a few tests. (GH-115953) --- Include/internal/pycore_optimizer.h | 85 +++++ Lib/test/test_generated_cases.py | 4 +- Lib/test/test_optimizer.py | 7 + Makefile.pre.in | 1 + Modules/_testinternalcapi.c | 3 +- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/optimizer_analysis.c | 343 +---------------- Python/optimizer_bytecodes.c | 170 ++++----- Python/optimizer_cases.c.h | 366 +++++++++---------- Python/optimizer_symbols.c | 332 +++++++++++++++++ Tools/c-analyzer/cpython/ignored.tsv | 2 - Tools/cases_generator/optimizer_generator.py | 6 +- 15 files changed, 720 insertions(+), 607 deletions(-) create mode 100644 Python/optimizer_symbols.c diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index eee71c700d4904..267cbf1265c420 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_uop_ids.h" +#include // This is the length of the trace we project initially. #define UOP_MAX_TRACE_LENGTH 512 @@ -25,6 +26,90 @@ extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; +/* Symbols */ + +struct _Py_UOpsSymType { + int flags; + PyTypeObject *typ; + // constant propagated value (might be NULL) + PyObject *const_val; +}; + +// Holds locals, stack, locals, stack ... co_consts (in that order) +#define MAX_ABSTRACT_INTERP_SIZE 4096 + +#define OVERALLOCATE_FACTOR 5 + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * OVERALLOCATE_FACTOR) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) + +typedef struct _Py_UOpsSymType _Py_UOpsSymType; + +struct _Py_UOpsAbstractFrame { + // Max stacklen + int stack_len; + int locals_len; + + _Py_UOpsSymType **stack_pointer; + _Py_UOpsSymType **stack; + _Py_UOpsSymType **locals; +}; + +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + _Py_UOpsSymType arena[TY_ARENA_SIZE]; +} ty_arena; + +struct _Py_UOpsAbstractInterpContext { + PyObject_HEAD + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + _Py_UOpsSymType **n_consumed; + _Py_UOpsSymType **limit; + _Py_UOpsSymType *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +}; + +typedef struct _Py_UOpsAbstractInterpContext _Py_UOpsAbstractInterpContext; + +extern bool _Py_uop_sym_is_null(_Py_UOpsSymType *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UOpsSymType *sym); +extern bool _Py_uop_sym_is_const(_Py_UOpsSymType *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UOpsSymType *sym); +extern _Py_UOpsSymType *_Py_uop_sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx); +extern _Py_UOpsSymType *_Py_uop_sym_new_not_null(_Py_UOpsAbstractInterpContext *ctx); +extern _Py_UOpsSymType *_Py_uop_sym_new_type( + _Py_UOpsAbstractInterpContext *ctx, PyTypeObject *typ); +extern _Py_UOpsSymType *_Py_uop_sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val); +extern _Py_UOpsSymType *_Py_uop_sym_new_null(_Py_UOpsAbstractInterpContext *ctx); +extern bool _Py_uop_sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ); +extern void _Py_uop_sym_set_null(_Py_UOpsSymType *sym); +extern void _Py_uop_sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp); + +extern int _Py_uop_abstractcontext_init(_Py_UOpsAbstractInterpContext *ctx); +extern void _Py_uop_abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_ctx_frame_new( + _Py_UOpsAbstractInterpContext *ctx, + PyCodeObject *co, + _Py_UOpsSymType **localsplus_start, + int n_locals_already_filled, + int curr_stackentries); +extern int _Py_uop_ctx_frame_pop(_Py_UOpsAbstractInterpContext *ctx); + +PyAPI_FUNC(PyObject *) +_Py_uop_symbols_test(PyObject *self, PyObject *ignored); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 6a5586822c2491..601775817043fe 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -900,7 +900,7 @@ def test_overridden_abstract_args(self): case OP2: { _Py_UOpsSymType *out; - out = sym_new_unknown(ctx); + out = _Py_uop_sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; @@ -925,7 +925,7 @@ def test_no_overridden_case(self): output = """ case OP: { _Py_UOpsSymType *out; - out = sym_new_unknown(ctx); + out = _Py_uop_sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index dfea8be3c6956f..899a4507317334 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -77,5 +77,12 @@ def func(x=0): _testinternalcapi.get_rare_event_counters()["func_modification"] ) + +class TestOptimizerSymbols(unittest.TestCase): + + def test_optimizer_symbols(self): + _testinternalcapi.uop_symbols_test() + + if __name__ == "__main__": unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index e4c64aea80d22c..7533a49b4392f0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -446,6 +446,7 @@ PYTHON_OBJS= \ Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ + Python/optimizer_symbols.o \ Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 5b714ca3f3dc33..9c6970d48ae6f7 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -24,6 +24,7 @@ #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_optimizer.h" // _Py_UOpsSymType, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -1677,7 +1678,6 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } #endif - static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1747,6 +1747,7 @@ static PyMethodDef module_functions[] = { #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif + {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 00ad3e2472af04..3a8a417a6bf47a 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -235,6 +235,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index aea5f730607658..5b34440af9322b 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -310,6 +310,9 @@ Source Files + + Python + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index ef21f85107bc32..88a4a7c9564309 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -601,6 +601,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ffe93dc787a8b8..27bd1121663398 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1382,6 +1382,9 @@ Python + + Python + Python diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e9751291e2fd5a..6b2aec8385824d 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -33,16 +33,6 @@ #include #include -// Holds locals, stack, locals, stack ... co_consts (in that order) -#define MAX_ABSTRACT_INTERP_SIZE 4096 - -#define OVERALLOCATE_FACTOR 5 - -#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * OVERALLOCATE_FACTOR) - -// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) - #ifdef Py_DEBUG extern const char *_PyUOpName(int index); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; @@ -61,318 +51,6 @@ #endif -// Flags for below. -#define KNOWN 1 << 0 -#define TRUE_CONST 1 << 1 -#define IS_NULL 1 << 2 -#define NOT_NULL 1 << 3 - -typedef struct { - int flags; - PyTypeObject *typ; - // constant propagated value (might be NULL) - PyObject *const_val; -} _Py_UOpsSymType; - - -typedef struct _Py_UOpsAbstractFrame { - // Max stacklen - int stack_len; - int locals_len; - - _Py_UOpsSymType **stack_pointer; - _Py_UOpsSymType **stack; - _Py_UOpsSymType **locals; -} _Py_UOpsAbstractFrame; - - -typedef struct ty_arena { - int ty_curr_number; - int ty_max_number; - _Py_UOpsSymType arena[TY_ARENA_SIZE]; -} ty_arena; - -// Tier 2 types meta interpreter -typedef struct _Py_UOpsAbstractInterpContext { - PyObject_HEAD - // The current "executing" frame. - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; - int curr_frame_depth; - - // Arena for the symbolic types. - ty_arena t_arena; - - _Py_UOpsSymType **n_consumed; - _Py_UOpsSymType **limit; - _Py_UOpsSymType *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; -} _Py_UOpsAbstractInterpContext; - -static inline _Py_UOpsSymType* sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx); - -// 0 on success, -1 on error. -static _Py_UOpsAbstractFrame * -ctx_frame_new( - _Py_UOpsAbstractInterpContext *ctx, - PyCodeObject *co, - _Py_UOpsSymType **localsplus_start, - int n_locals_already_filled, - int curr_stackentries -) -{ - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); - _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - - frame->stack_len = co->co_stacksize; - frame->locals_len = co->co_nlocalsplus; - - frame->locals = localsplus_start; - frame->stack = frame->locals + co->co_nlocalsplus; - frame->stack_pointer = frame->stack + curr_stackentries; - ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); - if (ctx->n_consumed >= ctx->limit) { - return NULL; - } - - - // Initialize with the initial state of all local variables - for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { - _Py_UOpsSymType *local = sym_new_unknown(ctx); - if (local == NULL) { - return NULL; - } - frame->locals[i] = local; - } - - - // Initialize the stack as well - for (int i = 0; i < curr_stackentries; i++) { - _Py_UOpsSymType *stackvar = sym_new_unknown(ctx); - if (stackvar == NULL) { - return NULL; - } - frame->stack[i] = stackvar; - } - - return frame; -} - -static void -abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx) -{ - if (ctx == NULL) { - return; - } - ctx->curr_frame_depth = 0; - int tys = ctx->t_arena.ty_curr_number; - for (int i = 0; i < tys; i++) { - Py_CLEAR(ctx->t_arena.arena[i].const_val); - } -} - -static int -abstractcontext_init( - _Py_UOpsAbstractInterpContext *ctx, - PyCodeObject *co, - int curr_stacklen, - int ir_entries -) -{ - ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; - ctx->n_consumed = ctx->locals_and_stack; -#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. - for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; - } -#endif - - // Setup the arena for sym expressions. - ctx->t_arena.ty_curr_number = 0; - ctx->t_arena.ty_max_number = TY_ARENA_SIZE; - - // Frame setup - ctx->curr_frame_depth = 0; - _Py_UOpsAbstractFrame *frame = ctx_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); - if (frame == NULL) { - return -1; - } - ctx->curr_frame_depth++; - ctx->frame = frame; - return 0; -} - - -static int -ctx_frame_pop( - _Py_UOpsAbstractInterpContext *ctx -) -{ - _Py_UOpsAbstractFrame *frame = ctx->frame; - - ctx->n_consumed = frame->locals; - ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; - - return 0; -} - - -// Takes a borrowed reference to const_val, turns that into a strong reference. -static _Py_UOpsSymType* -sym_new(_Py_UOpsAbstractInterpContext *ctx, - PyObject *const_val) -{ - _Py_UOpsSymType *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; - if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { - OPT_STAT_INC(optimizer_failure_reason_no_memory); - DPRINTF(1, "out of space for symbolic expression type\n"); - return NULL; - } - ctx->t_arena.ty_curr_number++; - self->const_val = NULL; - self->typ = NULL; - self->flags = 0; - - if (const_val != NULL) { - self->const_val = Py_NewRef(const_val); - } - - return self; -} - -static inline void -sym_set_flag(_Py_UOpsSymType *sym, int flag) -{ - sym->flags |= flag; -} - -static inline bool -sym_has_flag(_Py_UOpsSymType *sym, int flag) -{ - return (sym->flags & flag) != 0; -} - -static inline bool -sym_is_known(_Py_UOpsSymType *sym) -{ - return sym_has_flag(sym, KNOWN); -} - -static inline bool -sym_is_not_null(_Py_UOpsSymType *sym) -{ - return (sym->flags & (IS_NULL | NOT_NULL)) == NOT_NULL; -} - -static inline bool -sym_is_null(_Py_UOpsSymType *sym) -{ - return (sym->flags & (IS_NULL | NOT_NULL)) == IS_NULL; -} - -static inline bool -sym_is_const(_Py_UOpsSymType *sym) -{ - return (sym->flags & TRUE_CONST) != 0; -} - -static inline PyObject * -sym_get_const(_Py_UOpsSymType *sym) -{ - assert(sym_is_const(sym)); - assert(sym->const_val); - return sym->const_val; -} - -static inline void -sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp) -{ - assert(PyType_Check(tp)); - sym->typ = tp; - sym_set_flag(sym, KNOWN); - sym_set_flag(sym, NOT_NULL); -} - -static inline void -sym_set_null(_Py_UOpsSymType *sym) -{ - sym_set_flag(sym, IS_NULL); - sym_set_flag(sym, KNOWN); -} - - -static inline _Py_UOpsSymType* -sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx) -{ - return sym_new(ctx,NULL); -} - -static inline _Py_UOpsSymType* -sym_new_known_notnull(_Py_UOpsAbstractInterpContext *ctx) -{ - _Py_UOpsSymType *res = sym_new_unknown(ctx); - if (res == NULL) { - return NULL; - } - sym_set_flag(res, KNOWN); - sym_set_flag(res, NOT_NULL); - return res; -} - -static inline _Py_UOpsSymType* -sym_new_known_type(_Py_UOpsAbstractInterpContext *ctx, - PyTypeObject *typ) -{ - _Py_UOpsSymType *res = sym_new(ctx,NULL); - if (res == NULL) { - return NULL; - } - sym_set_type(res, typ); - return res; -} - -// Takes a borrowed reference to const_val. -static inline _Py_UOpsSymType* -sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) -{ - assert(const_val != NULL); - _Py_UOpsSymType *temp = sym_new( - ctx, - const_val - ); - if (temp == NULL) { - return NULL; - } - sym_set_type(temp, Py_TYPE(const_val)); - sym_set_flag(temp, TRUE_CONST); - sym_set_flag(temp, KNOWN); - sym_set_flag(temp, NOT_NULL); - return temp; -} - -static _Py_UOpsSymType* -sym_new_null(_Py_UOpsAbstractInterpContext *ctx) -{ - _Py_UOpsSymType *null_sym = sym_new_unknown(ctx); - if (null_sym == NULL) { - return NULL; - } - sym_set_null(null_sym); - return null_sym; -} - - -static inline bool -sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ) -{ - assert(typ == NULL || PyType_Check(typ)); - if (!sym_has_flag(sym, KNOWN)) { - return false; - } - return sym->typ == typ; -} - static inline bool op_is_end(uint32_t opcode) @@ -599,8 +277,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define _LOAD_ATTR_NOT_NULL \ do { \ - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); \ - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); \ + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); \ + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); \ } while (0); @@ -618,12 +296,15 @@ optimize_uops( _Py_UOpsAbstractInterpContext context; _Py_UOpsAbstractInterpContext *ctx = &context; - if (abstractcontext_init( - ctx, - co, curr_stacklen, - trace_len) < 0) { + if (_Py_uop_abstractcontext_init(ctx) < 0) { goto out_of_space; } + _Py_UOpsAbstractFrame *frame = _Py_uop_ctx_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; for (_PyUOpInstruction *this_instr = trace; this_instr < trace + trace_len && !op_is_end(this_instr->opcode); @@ -650,17 +331,17 @@ optimize_uops( assert(STACK_LEVEL() >= 0); } - abstractcontext_fini(ctx); + _Py_uop_abstractcontext_fini(ctx); return 1; out_of_space: DPRINTF(1, "Out of space in abstract interpreter\n"); - abstractcontext_fini(ctx); + _Py_uop_abstractcontext_fini(ctx); return 0; error: DPRINTF(1, "Encountered error in abstract interpreter\n"); - abstractcontext_fini(ctx); + _Py_uop_abstractcontext_fini(ctx); return 0; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b9afd3089e1077..e6e41f8968eaf3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -33,7 +33,7 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { + if (_Py_uop_sym_is_null(value)) { goto out_of_space; } } @@ -45,7 +45,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); _Py_UOpsSymType *temp; - OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(temp = _Py_uop_sym_new_null(ctx)); GETLOCAL(oparg) = temp; } @@ -54,147 +54,147 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = sym_new_null(ctx); + res = _Py_uop_sym_new_null(ctx); if (res == NULL) { goto out_of_space; }; } op(_GUARD_BOTH_INT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { + if (_Py_uop_sym_matches_type(left, &PyLong_Type) && + _Py_uop_sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); + _Py_uop_sym_set_type(left, &PyLong_Type); + _Py_uop_sym_set_type(right, &PyLong_Type); } op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { + if (_Py_uop_sym_matches_type(left, &PyFloat_Type) && + _Py_uop_sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); + _Py_uop_sym_set_type(left, &PyFloat_Type); + _Py_uop_sym_set_type(right, &PyFloat_Type); } op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - if (sym_matches_type(left, &PyUnicode_Type) && - sym_matches_type(right, &PyUnicode_Type)) { + if (_Py_uop_sym_matches_type(left, &PyUnicode_Type) && + _Py_uop_sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(right, &PyUnicode_Type); + _Py_uop_sym_set_type(left, &PyUnicode_Type); + _Py_uop_sym_set_type(right, &PyUnicode_Type); } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)_Py_uop_sym_get_const(left), + (PyLongObject *)_Py_uop_sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); } } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)_Py_uop_sym_get_const(left), + (PyLongObject *)_Py_uop_sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); } } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)_Py_uop_sym_get_const(left), + (PyLongObject *)_Py_uop_sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); } } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) + - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) + + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = _Py_uop_sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) - - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) - + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = _Py_uop_sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) * - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) * + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = _Py_uop_sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); } } @@ -205,21 +205,21 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); } op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); } @@ -240,8 +240,8 @@ dummy_func(void) { op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { (void)dict_version; - if (sym_is_const(owner)) { - PyObject *cnst = sym_get_const(owner); + if (_Py_uop_sym_is_const(owner)) { + PyObject *cnst = _Py_uop_sym_get_const(owner); if (PyModule_CheckExact(cnst)) { PyModuleObject *mod = (PyModuleObject *)cnst; PyObject *dict = mod->md_dict; @@ -257,23 +257,23 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. - assert(sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(_Py_uop_sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)_Py_uop_sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_const(ctx, res)); } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); } } @@ -297,38 +297,38 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); self = owner; } op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); self = owner; } op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); self = owner; } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { (void)callable; - OUT_OF_SPACE_IF_NULL(func = sym_new_known_notnull(ctx)); - OUT_OF_SPACE_IF_NULL(self = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(func = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = _Py_uop_sym_new_not_null(ctx)); } op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - sym_set_type(callable, &PyFunction_Type); + _Py_uop_sym_set_type(callable, &PyFunction_Type); (void)self_or_null; (void)func_version; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + _Py_uop_sym_set_null(null); + _Py_uop_sym_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -344,7 +344,7 @@ dummy_func(void) { assert(self_or_null != NULL); assert(args != NULL); - if (sym_is_not_null(self_or_null)) { + if (_Py_uop_sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; @@ -355,18 +355,18 @@ dummy_func(void) { // Can determine statically, so we interleave the new locals // and make the current stack the new locals. // This also sets up for true call inlining. - if (sym_is_known(self_or_null)) { + if (_Py_uop_sym_is_null(self_or_null) || _Py_uop_sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; } OUT_OF_SPACE_IF_NULL(new_frame = - ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + _Py_uop_ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); } op(_POP_FRAME, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx_frame_pop(ctx); + _Py_uop_ctx_frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = retval; } @@ -383,7 +383,7 @@ dummy_func(void) { /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); } } @@ -392,12 +392,12 @@ dummy_func(void) { (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); } } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - OUT_OF_SPACE_IF_NULL(next = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(next = _Py_uop_sym_new_type(ctx, &PyLong_Type)); (void)iter; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e8146dccbdfcd1..305158847da6fc 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -17,7 +17,7 @@ _Py_UOpsSymType *value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { + if (_Py_uop_sym_is_null(value)) { goto out_of_space; } stack_pointer[0] = value; @@ -37,7 +37,7 @@ _Py_UOpsSymType *value; value = GETLOCAL(oparg); _Py_UOpsSymType *temp; - OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(temp = _Py_uop_sym_new_null(ctx)); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -69,7 +69,7 @@ case _PUSH_NULL: { _Py_UOpsSymType *res; - res = sym_new_null(ctx); + res = _Py_uop_sym_new_null(ctx); if (res == NULL) { goto out_of_space; }; @@ -80,7 +80,7 @@ case _END_SEND: { _Py_UOpsSymType *value; - value = sym_new_unknown(ctx); + value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; stack_pointer += -1; @@ -89,7 +89,7 @@ case _UNARY_NEGATIVE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -97,7 +97,7 @@ case _UNARY_NOT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -105,7 +105,7 @@ case _TO_BOOL: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -117,7 +117,7 @@ case _TO_BOOL_INT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -125,7 +125,7 @@ case _TO_BOOL_LIST: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -133,7 +133,7 @@ case _TO_BOOL_NONE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -141,7 +141,7 @@ case _TO_BOOL_STR: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -149,7 +149,7 @@ case _TO_BOOL_ALWAYS_TRUE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -157,7 +157,7 @@ case _UNARY_INVERT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -168,12 +168,12 @@ _Py_UOpsSymType *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { + if (_Py_uop_sym_matches_type(left, &PyLong_Type) && + _Py_uop_sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); + _Py_uop_sym_set_type(left, &PyLong_Type); + _Py_uop_sym_set_type(right, &PyLong_Type); break; } @@ -183,20 +183,20 @@ _Py_UOpsSymType *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)_Py_uop_sym_get_const(left), + (PyLongObject *)_Py_uop_sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -209,20 +209,20 @@ _Py_UOpsSymType *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)_Py_uop_sym_get_const(left), + (PyLongObject *)_Py_uop_sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -235,20 +235,20 @@ _Py_UOpsSymType *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)_Py_uop_sym_get_const(left), + (PyLongObject *)_Py_uop_sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -260,12 +260,12 @@ _Py_UOpsSymType *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { + if (_Py_uop_sym_matches_type(left, &PyFloat_Type) && + _Py_uop_sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); + _Py_uop_sym_set_type(left, &PyFloat_Type); + _Py_uop_sym_set_type(right, &PyFloat_Type); break; } @@ -275,21 +275,21 @@ _Py_UOpsSymType *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) * - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) * + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = _Py_uop_sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -302,21 +302,21 @@ _Py_UOpsSymType *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) + - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) + + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = _Py_uop_sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -329,21 +329,21 @@ _Py_UOpsSymType *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); + assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) - - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) - + PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = _Py_uop_sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -355,18 +355,18 @@ _Py_UOpsSymType *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyUnicode_Type) && - sym_matches_type(right, &PyUnicode_Type)) { + if (_Py_uop_sym_matches_type(left, &PyUnicode_Type) && + _Py_uop_sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(right, &PyUnicode_Type); + _Py_uop_sym_set_type(left, &PyUnicode_Type); + _Py_uop_sym_set_type(right, &PyUnicode_Type); break; } case _BINARY_OP_ADD_UNICODE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -375,7 +375,7 @@ case _BINARY_SUBSCR: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -384,7 +384,7 @@ case _BINARY_SLICE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -398,7 +398,7 @@ case _BINARY_SUBSCR_LIST_INT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -407,7 +407,7 @@ case _BINARY_SUBSCR_STR_INT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -416,7 +416,7 @@ case _BINARY_SUBSCR_TUPLE_INT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -425,7 +425,7 @@ case _BINARY_SUBSCR_DICT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -466,7 +466,7 @@ case _CALL_INTRINSIC_1: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -474,7 +474,7 @@ case _CALL_INTRINSIC_2: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -487,7 +487,7 @@ retval = stack_pointer[-1]; stack_pointer += -1; ctx->frame->stack_pointer = stack_pointer; - ctx_frame_pop(ctx); + _Py_uop_ctx_frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = retval; stack_pointer[0] = res; @@ -501,7 +501,7 @@ case _GET_AITER: { _Py_UOpsSymType *iter; - iter = sym_new_unknown(ctx); + iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -509,7 +509,7 @@ case _GET_ANEXT: { _Py_UOpsSymType *awaitable; - awaitable = sym_new_unknown(ctx); + awaitable = _Py_uop_sym_new_unknown(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; stack_pointer += 1; @@ -518,7 +518,7 @@ case _GET_AWAITABLE: { _Py_UOpsSymType *iter; - iter = sym_new_unknown(ctx); + iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -537,7 +537,7 @@ case _LOAD_ASSERTION_ERROR: { _Py_UOpsSymType *value; - value = sym_new_unknown(ctx); + value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -546,7 +546,7 @@ case _LOAD_BUILD_CLASS: { _Py_UOpsSymType *bc; - bc = sym_new_unknown(ctx); + bc = _Py_uop_sym_new_unknown(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; stack_pointer += 1; @@ -570,7 +570,7 @@ /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); } stack_pointer += -1 + oparg; break; @@ -580,7 +580,7 @@ _Py_UOpsSymType **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = _Py_uop_sym_new_unknown(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -591,7 +591,7 @@ _Py_UOpsSymType **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = _Py_uop_sym_new_unknown(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -602,7 +602,7 @@ _Py_UOpsSymType **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = _Py_uop_sym_new_unknown(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -618,7 +618,7 @@ (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); } stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; @@ -645,7 +645,7 @@ case _LOAD_LOCALS: { _Py_UOpsSymType *locals; - locals = sym_new_unknown(ctx); + locals = _Py_uop_sym_new_unknown(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; stack_pointer += 1; @@ -654,7 +654,7 @@ case _LOAD_FROM_DICT_OR_GLOBALS: { _Py_UOpsSymType *v; - v = sym_new_unknown(ctx); + v = _Py_uop_sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; break; @@ -662,7 +662,7 @@ case _LOAD_NAME: { _Py_UOpsSymType *v; - v = sym_new_unknown(ctx); + v = _Py_uop_sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; stack_pointer += 1; @@ -672,9 +672,9 @@ case _LOAD_GLOBAL: { _Py_UOpsSymType *res; _Py_UOpsSymType *null = NULL; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - null = sym_new_null(ctx); + null = _Py_uop_sym_new_null(ctx); if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -693,9 +693,9 @@ case _LOAD_GLOBAL_MODULE: { _Py_UOpsSymType *res; _Py_UOpsSymType *null = NULL; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - null = sym_new_null(ctx); + null = _Py_uop_sym_new_null(ctx); if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -706,9 +706,9 @@ case _LOAD_GLOBAL_BUILTINS: { _Py_UOpsSymType *res; _Py_UOpsSymType *null = NULL; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - null = sym_new_null(ctx); + null = _Py_uop_sym_new_null(ctx); if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -730,7 +730,7 @@ case _LOAD_FROM_DICT_OR_DEREF: { _Py_UOpsSymType *value; - value = sym_new_unknown(ctx); + value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; @@ -738,7 +738,7 @@ case _LOAD_DEREF: { _Py_UOpsSymType *value; - value = sym_new_unknown(ctx); + value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -756,7 +756,7 @@ case _BUILD_STRING: { _Py_UOpsSymType *str; - str = sym_new_unknown(ctx); + str = _Py_uop_sym_new_unknown(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -765,7 +765,7 @@ case _BUILD_TUPLE: { _Py_UOpsSymType *tup; - tup = sym_new_unknown(ctx); + tup = _Py_uop_sym_new_unknown(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -774,7 +774,7 @@ case _BUILD_LIST: { _Py_UOpsSymType *list; - list = sym_new_unknown(ctx); + list = _Py_uop_sym_new_unknown(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -793,7 +793,7 @@ case _BUILD_SET: { _Py_UOpsSymType *set; - set = sym_new_unknown(ctx); + set = _Py_uop_sym_new_unknown(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -802,7 +802,7 @@ case _BUILD_MAP: { _Py_UOpsSymType *map; - map = sym_new_unknown(ctx); + map = _Py_uop_sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -815,7 +815,7 @@ case _BUILD_CONST_KEY_MAP: { _Py_UOpsSymType *map; - map = sym_new_unknown(ctx); + map = _Py_uop_sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -841,7 +841,7 @@ case _LOAD_SUPER_ATTR_ATTR: { _Py_UOpsSymType *attr; - attr = sym_new_unknown(ctx); + attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer += -2; @@ -851,9 +851,9 @@ case _LOAD_SUPER_ATTR_METHOD: { _Py_UOpsSymType *attr; _Py_UOpsSymType *self_or_null; - attr = sym_new_unknown(ctx); + attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); + self_or_null = _Py_uop_sym_new_unknown(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -864,9 +864,9 @@ case _LOAD_ATTR: { _Py_UOpsSymType *attr; _Py_UOpsSymType *self_or_null = NULL; - attr = sym_new_unknown(ctx); + attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); + self_or_null = _Py_uop_sym_new_unknown(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -902,8 +902,8 @@ owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand; (void)dict_version; - if (sym_is_const(owner)) { - PyObject *cnst = sym_get_const(owner); + if (_Py_uop_sym_is_const(owner)) { + PyObject *cnst = _Py_uop_sym_get_const(owner); if (PyModule_CheckExact(cnst)) { PyModuleObject *mod = (PyModuleObject *)cnst; PyObject *dict = mod->md_dict; @@ -925,23 +925,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. - assert(sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(_Py_uop_sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)_Py_uop_sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_const(ctx, res)); } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -1024,7 +1024,7 @@ case _COMPARE_OP: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1033,7 +1033,7 @@ case _COMPARE_OP_FLOAT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1042,7 +1042,7 @@ case _COMPARE_OP_INT: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1051,7 +1051,7 @@ case _COMPARE_OP_STR: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1060,7 +1060,7 @@ case _IS_OP: { _Py_UOpsSymType *b; - b = sym_new_unknown(ctx); + b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1069,7 +1069,7 @@ case _CONTAINS_OP: { _Py_UOpsSymType *b; - b = sym_new_unknown(ctx); + b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1079,9 +1079,9 @@ case _CHECK_EG_MATCH: { _Py_UOpsSymType *rest; _Py_UOpsSymType *match; - rest = sym_new_unknown(ctx); + rest = _Py_uop_sym_new_unknown(ctx); if (rest == NULL) goto out_of_space; - match = sym_new_unknown(ctx); + match = _Py_uop_sym_new_unknown(ctx); if (match == NULL) goto out_of_space; stack_pointer[-2] = rest; stack_pointer[-1] = match; @@ -1090,7 +1090,7 @@ case _CHECK_EXC_MATCH: { _Py_UOpsSymType *b; - b = sym_new_unknown(ctx); + b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1102,7 +1102,7 @@ case _IS_NONE: { _Py_UOpsSymType *b; - b = sym_new_unknown(ctx); + b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1110,7 +1110,7 @@ case _GET_LEN: { _Py_UOpsSymType *len_o; - len_o = sym_new_unknown(ctx); + len_o = _Py_uop_sym_new_unknown(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; stack_pointer += 1; @@ -1119,7 +1119,7 @@ case _MATCH_CLASS: { _Py_UOpsSymType *attrs; - attrs = sym_new_unknown(ctx); + attrs = _Py_uop_sym_new_unknown(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1128,7 +1128,7 @@ case _MATCH_MAPPING: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1137,7 +1137,7 @@ case _MATCH_SEQUENCE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1146,7 +1146,7 @@ case _MATCH_KEYS: { _Py_UOpsSymType *values_or_none; - values_or_none = sym_new_unknown(ctx); + values_or_none = _Py_uop_sym_new_unknown(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1155,7 +1155,7 @@ case _GET_ITER: { _Py_UOpsSymType *iter; - iter = sym_new_unknown(ctx); + iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1163,7 +1163,7 @@ case _GET_YIELD_FROM_ITER: { _Py_UOpsSymType *iter; - iter = sym_new_unknown(ctx); + iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1173,7 +1173,7 @@ case _FOR_ITER_TIER_TWO: { _Py_UOpsSymType *next; - next = sym_new_unknown(ctx); + next = _Py_uop_sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1194,7 +1194,7 @@ case _ITER_NEXT_LIST: { _Py_UOpsSymType *next; - next = sym_new_unknown(ctx); + next = _Py_uop_sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1213,7 +1213,7 @@ case _ITER_NEXT_TUPLE: { _Py_UOpsSymType *next; - next = sym_new_unknown(ctx); + next = _Py_uop_sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1234,7 +1234,7 @@ _Py_UOpsSymType *iter; _Py_UOpsSymType *next; iter = stack_pointer[-1]; - OUT_OF_SPACE_IF_NULL(next = sym_new_known_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(next = _Py_uop_sym_new_type(ctx, &PyLong_Type)); (void)iter; stack_pointer[0] = next; stack_pointer += 1; @@ -1246,9 +1246,9 @@ case _BEFORE_ASYNC_WITH: { _Py_UOpsSymType *exit; _Py_UOpsSymType *res; - exit = sym_new_unknown(ctx); + exit = _Py_uop_sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1259,9 +1259,9 @@ case _BEFORE_WITH: { _Py_UOpsSymType *exit; _Py_UOpsSymType *res; - exit = sym_new_unknown(ctx); + exit = _Py_uop_sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1271,7 +1271,7 @@ case _WITH_EXCEPT_START: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1281,9 +1281,9 @@ case _PUSH_EXC_INFO: { _Py_UOpsSymType *prev_exc; _Py_UOpsSymType *new_exc; - prev_exc = sym_new_unknown(ctx); + prev_exc = _Py_uop_sym_new_unknown(ctx); if (prev_exc == NULL) goto out_of_space; - new_exc = sym_new_unknown(ctx); + new_exc = _Py_uop_sym_new_unknown(ctx); if (new_exc == NULL) goto out_of_space; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -1306,7 +1306,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1321,7 +1321,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1331,7 +1331,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _Py_UOpsSymType *attr; - attr = sym_new_unknown(ctx); + attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1339,7 +1339,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _Py_UOpsSymType *attr; - attr = sym_new_unknown(ctx); + attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1356,7 +1356,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1373,8 +1373,8 @@ _Py_UOpsSymType *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + _Py_uop_sym_set_null(null); + _Py_uop_sym_set_type(callable, &PyMethod_Type); break; } @@ -1384,8 +1384,8 @@ _Py_UOpsSymType *self; callable = stack_pointer[-2 - oparg]; (void)callable; - OUT_OF_SPACE_IF_NULL(func = sym_new_known_notnull(ctx)); - OUT_OF_SPACE_IF_NULL(self = sym_new_known_notnull(ctx)); + OUT_OF_SPACE_IF_NULL(func = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = _Py_uop_sym_new_not_null(ctx)); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -1401,7 +1401,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand; - sym_set_type(callable, &PyFunction_Type); + _Py_uop_sym_set_type(callable, &PyFunction_Type); (void)self_or_null; (void)func_version; break; @@ -1428,7 +1428,7 @@ PyCodeObject *co = (PyCodeObject *)func->func_code; assert(self_or_null != NULL); assert(args != NULL); - if (sym_is_not_null(self_or_null)) { + if (_Py_uop_sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; @@ -1438,12 +1438,12 @@ // Can determine statically, so we interleave the new locals // and make the current stack the new locals. // This also sets up for true call inlining. - if (sym_is_known(self_or_null)) { + if (_Py_uop_sym_is_null(self_or_null) || _Py_uop_sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; } OUT_OF_SPACE_IF_NULL(new_frame = - ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + _Py_uop_ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); stack_pointer[-2 - oparg] = (_Py_UOpsSymType *)new_frame; stack_pointer += -1 - oparg; break; @@ -1464,7 +1464,7 @@ case _CALL_TYPE_1: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1473,7 +1473,7 @@ case _CALL_STR_1: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1482,7 +1482,7 @@ case _CALL_TUPLE_1: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1498,7 +1498,7 @@ case _CALL_BUILTIN_CLASS: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1507,7 +1507,7 @@ case _CALL_BUILTIN_O: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1516,7 +1516,7 @@ case _CALL_BUILTIN_FAST: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1525,7 +1525,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1534,7 +1534,7 @@ case _CALL_LEN: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1543,7 +1543,7 @@ case _CALL_ISINSTANCE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1552,7 +1552,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1561,7 +1561,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1570,7 +1570,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1579,7 +1579,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1596,7 +1596,7 @@ case _MAKE_FUNCTION: { _Py_UOpsSymType *func; - func = sym_new_unknown(ctx); + func = _Py_uop_sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; break; @@ -1604,7 +1604,7 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UOpsSymType *func; - func = sym_new_unknown(ctx); + func = _Py_uop_sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; stack_pointer += -1; @@ -1613,7 +1613,7 @@ case _BUILD_SLICE: { _Py_UOpsSymType *slice; - slice = sym_new_unknown(ctx); + slice = _Py_uop_sym_new_unknown(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1622,7 +1622,7 @@ case _CONVERT_VALUE: { _Py_UOpsSymType *result; - result = sym_new_unknown(ctx); + result = _Py_uop_sym_new_unknown(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; break; @@ -1630,7 +1630,7 @@ case _FORMAT_SIMPLE: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -1638,7 +1638,7 @@ case _FORMAT_WITH_SPEC: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1658,7 +1658,7 @@ case _BINARY_OP: { _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); + res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1732,7 +1732,7 @@ case _LOAD_CONST_INLINE: { _Py_UOpsSymType *value; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; break; @@ -1741,7 +1741,7 @@ case _LOAD_CONST_INLINE_BORROW: { _Py_UOpsSymType *value; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; break; @@ -1751,8 +1751,8 @@ _Py_UOpsSymType *value; _Py_UOpsSymType *null; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -1763,8 +1763,8 @@ _Py_UOpsSymType *value; _Py_UOpsSymType *null; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c new file mode 100644 index 00000000000000..1b7c88cf785fd2 --- /dev/null +++ b/Python/optimizer_symbols.c @@ -0,0 +1,332 @@ + +#include "Python.h" + +#include "cpython/optimizer.h" +#include "pycore_code.h" +#include "pycore_frame.h" +#include "pycore_optimizer.h" + +#include +#include +#include + +// Flags for below. +#define KNOWN 1 << 0 +#define TRUE_CONST 1 << 1 +#define IS_NULL 1 << 2 +#define NOT_NULL 1 << 3 + +#ifdef Py_DEBUG +static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV("PYTHON_OPT_DEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; +} +#define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else +#define DPRINTF(level, ...) +#endif + +// Takes a borrowed reference to const_val, turns that into a strong reference. +static _Py_UOpsSymType* +sym_new(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) +{ + _Py_UOpsSymType *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { + OPT_STAT_INC(optimizer_failure_reason_no_memory); + DPRINTF(1, "out of space for symbolic expression type\n"); + return NULL; + } + ctx->t_arena.ty_curr_number++; + self->const_val = NULL; + self->typ = NULL; + self->flags = 0; + + if (const_val != NULL) { + self->const_val = Py_NewRef(const_val); + } + + return self; +} + +static inline void +sym_set_flag(_Py_UOpsSymType *sym, int flag) +{ + sym->flags |= flag; +} + +static inline bool +sym_has_flag(_Py_UOpsSymType *sym, int flag) +{ + return (sym->flags & flag) != 0; +} + +bool +_Py_uop_sym_is_not_null(_Py_UOpsSymType *sym) +{ + return (sym->flags & (IS_NULL | NOT_NULL)) == NOT_NULL; +} + +bool +_Py_uop_sym_is_null(_Py_UOpsSymType *sym) +{ + return (sym->flags & (IS_NULL | NOT_NULL)) == IS_NULL; +} + +bool +_Py_uop_sym_is_const(_Py_UOpsSymType *sym) +{ + return (sym->flags & TRUE_CONST) != 0; +} + +PyObject * +_Py_uop_sym_get_const(_Py_UOpsSymType *sym) +{ + assert(_Py_uop_sym_is_const(sym)); + assert(sym->const_val); + return sym->const_val; +} + +void +_Py_uop_sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp) +{ + assert(PyType_Check(tp)); + sym->typ = tp; + sym_set_flag(sym, KNOWN); + sym_set_flag(sym, NOT_NULL); +} + +void +_Py_uop_sym_set_null(_Py_UOpsSymType *sym) +{ + sym_set_flag(sym, IS_NULL); + sym_set_flag(sym, KNOWN); +} + + +_Py_UOpsSymType * +_Py_uop_sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx) +{ + return sym_new(ctx,NULL); +} + +_Py_UOpsSymType * +_Py_uop_sym_new_not_null(_Py_UOpsAbstractInterpContext *ctx) +{ + _Py_UOpsSymType *res = _Py_uop_sym_new_unknown(ctx); + if (res == NULL) { + return NULL; + } + sym_set_flag(res, KNOWN); + sym_set_flag(res, NOT_NULL); + return res; +} + +_Py_UOpsSymType * +_Py_uop_sym_new_type(_Py_UOpsAbstractInterpContext *ctx, + PyTypeObject *typ) +{ + _Py_UOpsSymType *res = sym_new(ctx,NULL); + if (res == NULL) { + return NULL; + } + _Py_uop_sym_set_type(res, typ); + return res; +} + +// Takes a borrowed reference to const_val. +_Py_UOpsSymType* +_Py_uop_sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + _Py_UOpsSymType *temp = sym_new( + ctx, + const_val + ); + if (temp == NULL) { + return NULL; + } + _Py_uop_sym_set_type(temp, Py_TYPE(const_val)); + sym_set_flag(temp, TRUE_CONST); + sym_set_flag(temp, KNOWN); + sym_set_flag(temp, NOT_NULL); + return temp; +} + +_Py_UOpsSymType* +_Py_uop_sym_new_null(_Py_UOpsAbstractInterpContext *ctx) +{ + _Py_UOpsSymType *null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym == NULL) { + return NULL; + } + _Py_uop_sym_set_null(null_sym); + return null_sym; +} + +bool +_Py_uop_sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ) +{ + assert(typ == NULL || PyType_Check(typ)); + if (!sym_has_flag(sym, KNOWN)) { + return false; + } + return sym->typ == typ; +} + +// 0 on success, -1 on error. +_Py_UOpsAbstractFrame * +_Py_uop_ctx_frame_new( + _Py_UOpsAbstractInterpContext *ctx, + PyCodeObject *co, + _Py_UOpsSymType **localsplus_start, + int n_locals_already_filled, + int curr_stackentries +) +{ + assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + + frame->stack_len = co->co_stacksize; + frame->locals_len = co->co_nlocalsplus; + + frame->locals = localsplus_start; + frame->stack = frame->locals + co->co_nlocalsplus; + frame->stack_pointer = frame->stack + curr_stackentries; + ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); + if (ctx->n_consumed >= ctx->limit) { + return NULL; + } + + + // Initialize with the initial state of all local variables + for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { + _Py_UOpsSymType *local = _Py_uop_sym_new_unknown(ctx); + if (local == NULL) { + return NULL; + } + frame->locals[i] = local; + } + + + // Initialize the stack as well + for (int i = 0; i < curr_stackentries; i++) { + _Py_UOpsSymType *stackvar = _Py_uop_sym_new_unknown(ctx); + if (stackvar == NULL) { + return NULL; + } + frame->stack[i] = stackvar; + } + + return frame; +} + +void +_Py_uop_abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx) +{ + if (ctx == NULL) { + return; + } + ctx->curr_frame_depth = 0; + int tys = ctx->t_arena.ty_curr_number; + for (int i = 0; i < tys; i++) { + Py_CLEAR(ctx->t_arena.arena[i].const_val); + } +} + +int +_Py_uop_abstractcontext_init( + _Py_UOpsAbstractInterpContext *ctx +) +{ + ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; + ctx->n_consumed = ctx->locals_and_stack; +#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. + for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { + ctx->locals_and_stack[i] = NULL; + } +#endif + + // Setup the arena for sym expressions. + ctx->t_arena.ty_curr_number = 0; + ctx->t_arena.ty_max_number = TY_ARENA_SIZE; + + // Frame setup + ctx->curr_frame_depth = 0; + + return 0; +} + +int +_Py_uop_ctx_frame_pop( + _Py_UOpsAbstractInterpContext *ctx +) +{ + _Py_UOpsAbstractFrame *frame = ctx->frame; + + ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; + assert(ctx->curr_frame_depth >= 1); + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + return 0; +} + +#define TEST_PREDICATE(PRED, MSG) \ +do { \ + if (!(PRED)) { \ + PyErr_SetString( \ + PyExc_AssertionError, \ + (MSG)); \ + goto fail; \ + } \ +} while (0) + +/* +static _Py_UOpsSymType * +make_bottom(_Py_UOpsAbstractInterpContext *ctx) +{ + _Py_UOpsSymType *sym = _Py_uop_sym_new_unknown(ctx); + _Py_uop_sym_set_null(sym); + _Py_uop_sym_set_type(sym, &PyLong_Type); + return sym; +}*/ + +PyObject * +_Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) +{ + _Py_UOpsAbstractInterpContext context; + _Py_UOpsAbstractInterpContext *ctx = &context; + _Py_uop_abstractcontext_init(ctx); + + _Py_UOpsSymType *top = _Py_uop_sym_new_unknown(ctx); + if (top == NULL) { + return NULL; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(top), "unknown is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(top), "unknown is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_const(top), "unknown is a constant"); + // TEST_PREDICATE(_Py_uop_sym_get_const(top) == NULL, "unknown as constant is not NULL"); + + // _Py_UOpsSymType *bottom = make_bottom(ctx); + // TEST_PREDICATE(_Py_uop_sym_is_null(bottom), "bottom is NULL is not true"); + // TEST_PREDICATE(_Py_uop_sym_is_not_null(bottom), "bottom is not NULL is not true"); + // TEST_PREDICATE(_Py_uop_sym_is_const(bottom), "bottom is a constant is not true"); + + _Py_UOpsSymType *int_type = _Py_uop_sym_new_type(ctx, &PyLong_Type); + TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "inconsistent type"); + _Py_uop_sym_set_type(int_type, &PyLong_Type); + TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "inconsistent type"); + _Py_uop_sym_set_type(int_type, &PyFloat_Type); + // TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "bottom doesn't match int"); + + _Py_uop_abstractcontext_fini(ctx); + Py_RETURN_NONE; +fail: + _Py_uop_abstractcontext_fini(ctx); + return NULL; +} diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index e9b81cf4c7d653..0f212ecb528ac4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -740,6 +740,4 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - -Python/optimizer_analysis.c - _Py_UOpsAbstractFrame_Type - -Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - Modules/clinic/md5module.c.h _md5_md5 _keywords - diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index aa3f4ecb11d58b..f110a74e3ad351 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -87,14 +87,14 @@ def emit_default(out: CWriter, uop: Uop) -> None: if var.name != "unused" and not var.peek: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_unknown(ctx);\n") + out.emit(f"{var.name}[_i] = _Py_uop_sym_new_unknown(ctx);\n") out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") out.emit("}\n") elif var.name == "null": - out.emit(f"{var.name} = sym_new_null(ctx);\n") + out.emit(f"{var.name} = _Py_uop_sym_new_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") else: - out.emit(f"{var.name} = sym_new_unknown(ctx);\n") + out.emit(f"{var.name} = _Py_uop_sym_new_unknown(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") From 6ecfcfe8946cf701c6c7018e30ae54d8b7a7ac2a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 27 Feb 2024 13:25:02 +0000 Subject: [PATCH 20/82] GH-115816: Assorted naming and formatting changes to improve maintainability. (GH-115987) * Rename _Py_UOpsAbstractInterpContext to _Py_UOpsContext and _Py_UOpsSymType to _Py_UopsSymbol. * #define shortened form of _Py_uop_... names for improved readability. --- Include/internal/pycore_optimizer.h | 73 ++- Lib/test/test_generated_cases.py | 12 +- Modules/_testinternalcapi.c | 2 +- Python/optimizer_analysis.c | 25 +- Python/optimizer_bytecodes.c | 213 ++++---- Python/optimizer_cases.c.h | 538 +++++++++---------- Python/optimizer_symbols.c | 105 ++-- Tools/cases_generator/optimizer_generator.py | 6 +- 8 files changed, 498 insertions(+), 476 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 267cbf1265c420..425bd693fac53d 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -28,7 +28,7 @@ extern PyTypeObject _PyUOpOptimizer_Type; /* Symbols */ -struct _Py_UOpsSymType { +struct _Py_UopsSymbol { int flags; PyTypeObject *typ; // constant propagated value (might be NULL) @@ -38,23 +38,21 @@ struct _Py_UOpsSymType { // Holds locals, stack, locals, stack ... co_consts (in that order) #define MAX_ABSTRACT_INTERP_SIZE 4096 -#define OVERALLOCATE_FACTOR 5 - -#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * OVERALLOCATE_FACTOR) +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) #define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) -typedef struct _Py_UOpsSymType _Py_UOpsSymType; +typedef struct _Py_UopsSymbol _Py_UopsSymbol; struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - _Py_UOpsSymType **stack_pointer; - _Py_UOpsSymType **stack; - _Py_UOpsSymType **locals; + _Py_UopsSymbol **stack_pointer; + _Py_UopsSymbol **stack; + _Py_UopsSymbol **locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -62,10 +60,10 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; typedef struct ty_arena { int ty_curr_number; int ty_max_number; - _Py_UOpsSymType arena[TY_ARENA_SIZE]; + _Py_UopsSymbol arena[TY_ARENA_SIZE]; } ty_arena; -struct _Py_UOpsAbstractInterpContext { +struct _Py_UOpsContext { PyObject_HEAD // The current "executing" frame. _Py_UOpsAbstractFrame *frame; @@ -75,40 +73,39 @@ struct _Py_UOpsAbstractInterpContext { // Arena for the symbolic types. ty_arena t_arena; - _Py_UOpsSymType **n_consumed; - _Py_UOpsSymType **limit; - _Py_UOpsSymType *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + _Py_UopsSymbol **n_consumed; + _Py_UopsSymbol **limit; + _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; }; -typedef struct _Py_UOpsAbstractInterpContext _Py_UOpsAbstractInterpContext; - -extern bool _Py_uop_sym_is_null(_Py_UOpsSymType *sym); -extern bool _Py_uop_sym_is_not_null(_Py_UOpsSymType *sym); -extern bool _Py_uop_sym_is_const(_Py_UOpsSymType *sym); -extern PyObject *_Py_uop_sym_get_const(_Py_UOpsSymType *sym); -extern _Py_UOpsSymType *_Py_uop_sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx); -extern _Py_UOpsSymType *_Py_uop_sym_new_not_null(_Py_UOpsAbstractInterpContext *ctx); -extern _Py_UOpsSymType *_Py_uop_sym_new_type( - _Py_UOpsAbstractInterpContext *ctx, PyTypeObject *typ); -extern _Py_UOpsSymType *_Py_uop_sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val); -extern _Py_UOpsSymType *_Py_uop_sym_new_null(_Py_UOpsAbstractInterpContext *ctx); -extern bool _Py_uop_sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ); -extern void _Py_uop_sym_set_null(_Py_UOpsSymType *sym); -extern void _Py_uop_sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp); - -extern int _Py_uop_abstractcontext_init(_Py_UOpsAbstractInterpContext *ctx); -extern void _Py_uop_abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx); - -extern _Py_UOpsAbstractFrame *_Py_uop_ctx_frame_new( - _Py_UOpsAbstractInterpContext *ctx, +typedef struct _Py_UOpsContext _Py_UOpsContext; + +extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_type( + _Py_UOpsContext *ctx, PyTypeObject *typ); +extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern void _Py_uop_sym_set_null(_Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *tp); + +extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( + _Py_UOpsContext *ctx, PyCodeObject *co, - _Py_UOpsSymType **localsplus_start, + _Py_UopsSymbol **localsplus_start, int n_locals_already_filled, int curr_stackentries); -extern int _Py_uop_ctx_frame_pop(_Py_UOpsAbstractInterpContext *ctx); +extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); -PyAPI_FUNC(PyObject *) -_Py_uop_symbols_test(PyObject *self, PyObject *ignored); +PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); #ifdef __cplusplus } diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 601775817043fe..18bf8ab29148c4 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -890,8 +890,8 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - _Py_UOpsSymType *arg1; - _Py_UOpsSymType *out; + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; arg1 = stack_pointer[-1]; eggs(); stack_pointer[-1] = out; @@ -899,7 +899,7 @@ def test_overridden_abstract_args(self): } case OP2: { - _Py_UOpsSymType *out; + _Py_UopsSymbol *out; out = _Py_uop_sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; @@ -924,7 +924,7 @@ def test_no_overridden_case(self): """ output = """ case OP: { - _Py_UOpsSymType *out; + _Py_UopsSymbol *out; out = _Py_uop_sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; @@ -932,8 +932,8 @@ def test_no_overridden_case(self): } case OP2: { - _Py_UOpsSymType *arg1; - _Py_UOpsSymType *out; + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; arg1 = stack_pointer[-1]; stack_pointer[-1] = out; break; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 9c6970d48ae6f7..db8817418950b9 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -24,7 +24,7 @@ #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() -#include "pycore_optimizer.h" // _Py_UOpsSymType, etc. +#include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6b2aec8385824d..b29a00c941e996 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -282,6 +282,23 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, } while (0); +/* Shortened forms for convenience, used in optimizer_bytecodes.c */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_set_null _Py_uop_sym_set_null +#define sym_set_type _Py_uop_sym_set_type +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( @@ -293,13 +310,13 @@ optimize_uops( ) { - _Py_UOpsAbstractInterpContext context; - _Py_UOpsAbstractInterpContext *ctx = &context; + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; if (_Py_uop_abstractcontext_init(ctx) < 0) { goto out_of_space; } - _Py_UOpsAbstractFrame *frame = _Py_uop_ctx_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); if (frame == NULL) { return -1; } @@ -313,7 +330,7 @@ optimize_uops( int oparg = this_instr->oparg; uint32_t opcode = this_instr->opcode; - _Py_UOpsSymType **stack_pointer = ctx->frame->stack_pointer; + _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; DPRINTF(3, "Abstract interpreting %s:%d ", _PyUOpName(opcode), diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e6e41f8968eaf3..68737389c66b67 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1,29 +1,46 @@ #include "Python.h" +#include "pycore_optimizer.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" #include "internal/pycore_moduleobject.h" #define op(name, ...) /* NAME is ignored */ -typedef struct _Py_UOpsSymType _Py_UOpsSymType; -typedef struct _Py_UOpsAbstractInterpContext _Py_UOpsAbstractInterpContext; +typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UOpsContext _Py_UOpsContext; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_set_null _Py_uop_sym_set_null +#define sym_set_type _Py_uop_sym_set_type +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + static int dummy_func(void) { PyCodeObject *code; int oparg; - _Py_UOpsSymType *flag; - _Py_UOpsSymType *left; - _Py_UOpsSymType *right; - _Py_UOpsSymType *value; - _Py_UOpsSymType *res; - _Py_UOpsSymType *iter; - _Py_UOpsSymType *top; - _Py_UOpsSymType *bottom; + _Py_UopsSymbol *flag; + _Py_UopsSymbol *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractInterpContext *ctx; + _Py_UOpsContext *ctx; _PyUOpInstruction *this_instr; _PyBloomFilter *dependencies; int modified; @@ -33,7 +50,7 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (_Py_uop_sym_is_null(value)) { + if (sym_is_null(value)) { goto out_of_space; } } @@ -44,8 +61,8 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UOpsSymType *temp; - OUT_OF_SPACE_IF_NULL(temp = _Py_uop_sym_new_null(ctx)); + _Py_UopsSymbol *temp; + OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); GETLOCAL(oparg) = temp; } @@ -54,147 +71,147 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = _Py_uop_sym_new_null(ctx); + res = sym_new_null(ctx); if (res == NULL) { goto out_of_space; }; } op(_GUARD_BOTH_INT, (left, right -- left, right)) { - if (_Py_uop_sym_matches_type(left, &PyLong_Type) && - _Py_uop_sym_matches_type(right, &PyLong_Type)) { + if (sym_matches_type(left, &PyLong_Type) && + sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - _Py_uop_sym_set_type(left, &PyLong_Type); - _Py_uop_sym_set_type(right, &PyLong_Type); + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); } op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - if (_Py_uop_sym_matches_type(left, &PyFloat_Type) && - _Py_uop_sym_matches_type(right, &PyFloat_Type)) { + if (sym_matches_type(left, &PyFloat_Type) && + sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - _Py_uop_sym_set_type(left, &PyFloat_Type); - _Py_uop_sym_set_type(right, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); } op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - if (_Py_uop_sym_matches_type(left, &PyUnicode_Type) && - _Py_uop_sym_matches_type(right, &PyUnicode_Type)) { + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - _Py_uop_sym_set_type(left, &PyUnicode_Type); - _Py_uop_sym_set_type(right, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + sym_set_type(right, &PyUnicode_Type); } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)_Py_uop_sym_get_const(left), - (PyLongObject *)_Py_uop_sym_get_const(right)); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)_Py_uop_sym_get_const(left), - (PyLongObject *)_Py_uop_sym_get_const(right)); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)_Py_uop_sym_get_const(left), - (PyLongObject *)_Py_uop_sym_get_const(right)); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) + - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); if (temp == NULL) { goto error; } - res = _Py_uop_sym_new_const(ctx, temp); + res = sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) - - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); if (temp == NULL) { goto error; } - res = _Py_uop_sym_new_const(ctx, temp); + res = sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) * - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); if (temp == NULL) { goto error; } - res = _Py_uop_sym_new_const(ctx, temp); + res = sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); } } @@ -205,21 +222,21 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); } op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); } @@ -240,8 +257,8 @@ dummy_func(void) { op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { (void)dict_version; - if (_Py_uop_sym_is_const(owner)) { - PyObject *cnst = _Py_uop_sym_get_const(owner); + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); if (PyModule_CheckExact(cnst)) { PyModuleObject *mod = (PyModuleObject *)cnst; PyObject *dict = mod->md_dict; @@ -257,23 +274,23 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. - assert(_Py_uop_sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)_Py_uop_sym_get_const(owner); + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_const(ctx, res)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); } } @@ -297,38 +314,38 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); self = owner; } op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); self = owner; } op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); self = owner; } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { (void)callable; - OUT_OF_SPACE_IF_NULL(func = _Py_uop_sym_new_not_null(ctx)); - OUT_OF_SPACE_IF_NULL(self = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); } op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - _Py_uop_sym_set_type(callable, &PyFunction_Type); + sym_set_type(callable, &PyFunction_Type); (void)self_or_null; (void)func_version; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - _Py_uop_sym_set_null(null); - _Py_uop_sym_set_type(callable, &PyMethod_Type); + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -344,29 +361,29 @@ dummy_func(void) { assert(self_or_null != NULL); assert(args != NULL); - if (_Py_uop_sym_is_not_null(self_or_null)) { + if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - _Py_UOpsSymType **localsplus_start = ctx->n_consumed; + _Py_UopsSymbol **localsplus_start = ctx->n_consumed; int n_locals_already_filled = 0; // Can determine statically, so we interleave the new locals // and make the current stack the new locals. // This also sets up for true call inlining. - if (_Py_uop_sym_is_null(self_or_null) || _Py_uop_sym_is_not_null(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; } OUT_OF_SPACE_IF_NULL(new_frame = - _Py_uop_ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); } op(_POP_FRAME, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - _Py_uop_ctx_frame_pop(ctx); + frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = retval; } @@ -383,7 +400,7 @@ dummy_func(void) { /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); } } @@ -392,12 +409,12 @@ dummy_func(void) { (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); } } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - OUT_OF_SPACE_IF_NULL(next = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); (void)iter; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 305158847da6fc..9e99c8395a405b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -14,10 +14,10 @@ /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (_Py_uop_sym_is_null(value)) { + if (sym_is_null(value)) { goto out_of_space; } stack_pointer[0] = value; @@ -26,7 +26,7 @@ } case _LOAD_FAST: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -34,10 +34,10 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); - _Py_UOpsSymType *temp; - OUT_OF_SPACE_IF_NULL(temp = _Py_uop_sym_new_null(ctx)); + _Py_UopsSymbol *temp; + OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -45,7 +45,7 @@ } case _LOAD_CONST: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; // There should be no LOAD_CONST. It should be all // replaced by peephole_opt. Py_UNREACHABLE(); @@ -55,7 +55,7 @@ } case _STORE_FAST: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -68,8 +68,8 @@ } case _PUSH_NULL: { - _Py_UOpsSymType *res; - res = _Py_uop_sym_new_null(ctx); + _Py_UopsSymbol *res; + res = sym_new_null(ctx); if (res == NULL) { goto out_of_space; }; @@ -79,7 +79,7 @@ } case _END_SEND: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; @@ -88,7 +88,7 @@ } case _UNARY_NEGATIVE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -96,7 +96,7 @@ } case _UNARY_NOT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -104,7 +104,7 @@ } case _TO_BOOL: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -116,7 +116,7 @@ } case _TO_BOOL_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -124,7 +124,7 @@ } case _TO_BOOL_LIST: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -132,7 +132,7 @@ } case _TO_BOOL_NONE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -140,7 +140,7 @@ } case _TO_BOOL_STR: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -148,7 +148,7 @@ } case _TO_BOOL_ALWAYS_TRUE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -156,7 +156,7 @@ } case _UNARY_INVERT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -164,39 +164,39 @@ } case _GUARD_BOTH_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_matches_type(left, &PyLong_Type) && - _Py_uop_sym_matches_type(right, &PyLong_Type)) { + if (sym_matches_type(left, &PyLong_Type) && + sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - _Py_uop_sym_set_type(left, &PyLong_Type); - _Py_uop_sym_set_type(right, &PyLong_Type); + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); break; } case _BINARY_OP_MULTIPLY_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)_Py_uop_sym_get_const(left), - (PyLongObject *)_Py_uop_sym_get_const(right)); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -204,25 +204,25 @@ } case _BINARY_OP_ADD_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)_Py_uop_sym_get_const(left), - (PyLongObject *)_Py_uop_sym_get_const(right)); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -230,25 +230,25 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyLong_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyLong_CheckExact(_Py_uop_sym_get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)_Py_uop_sym_get_const(left), - (PyLongObject *)_Py_uop_sym_get_const(right)); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_const(ctx, temp)); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -256,40 +256,40 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_matches_type(left, &PyFloat_Type) && - _Py_uop_sym_matches_type(right, &PyFloat_Type)) { + if (sym_matches_type(left, &PyFloat_Type) && + sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - _Py_uop_sym_set_type(left, &PyFloat_Type); - _Py_uop_sym_set_type(right, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) * - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); if (temp == NULL) { goto error; } - res = _Py_uop_sym_new_const(ctx, temp); + res = sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -297,26 +297,26 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) + - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); if (temp == NULL) { goto error; } - res = _Py_uop_sym_new_const(ctx, temp); + res = sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -324,26 +324,26 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_is_const(left) && _Py_uop_sym_is_const(right)) { - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(left))); - assert(PyFloat_CheckExact(_Py_uop_sym_get_const(right))); + if (sym_is_const(left) && sym_is_const(right)) { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(left)) - - PyFloat_AS_DOUBLE(_Py_uop_sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); if (temp == NULL) { goto error; } - res = _Py_uop_sym_new_const(ctx, temp); + res = sym_new_const(ctx, temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_type(ctx, &PyFloat_Type)); + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -351,21 +351,21 @@ } case _GUARD_BOTH_UNICODE: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (_Py_uop_sym_matches_type(left, &PyUnicode_Type) && - _Py_uop_sym_matches_type(right, &PyUnicode_Type)) { + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - _Py_uop_sym_set_type(left, &PyUnicode_Type); - _Py_uop_sym_set_type(right, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + sym_set_type(right, &PyUnicode_Type); break; } case _BINARY_OP_ADD_UNICODE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -374,7 +374,7 @@ } case _BINARY_SUBSCR: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -383,7 +383,7 @@ } case _BINARY_SLICE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; @@ -397,7 +397,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -406,7 +406,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -415,7 +415,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -424,7 +424,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -465,7 +465,7 @@ } case _CALL_INTRINSIC_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -473,7 +473,7 @@ } case _CALL_INTRINSIC_2: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -482,12 +482,12 @@ } case _POP_FRAME: { - _Py_UOpsSymType *retval; - _Py_UOpsSymType *res; + _Py_UopsSymbol *retval; + _Py_UopsSymbol *res; retval = stack_pointer[-1]; stack_pointer += -1; ctx->frame->stack_pointer = stack_pointer; - _Py_uop_ctx_frame_pop(ctx); + frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = retval; stack_pointer[0] = res; @@ -500,7 +500,7 @@ /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ case _GET_AITER: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -508,7 +508,7 @@ } case _GET_ANEXT: { - _Py_UOpsSymType *awaitable; + _Py_UopsSymbol *awaitable; awaitable = _Py_uop_sym_new_unknown(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; @@ -517,7 +517,7 @@ } case _GET_AWAITABLE: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -536,7 +536,7 @@ } case _LOAD_ASSERTION_ERROR: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; @@ -545,7 +545,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UOpsSymType *bc; + _Py_UopsSymbol *bc; bc = _Py_uop_sym_new_unknown(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; @@ -563,21 +563,21 @@ } case _UNPACK_SEQUENCE: { - _Py_UOpsSymType *seq; - _Py_UOpsSymType **values; + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); } stack_pointer += -1 + oparg; break; } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UOpsSymType **values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = _Py_uop_sym_new_unknown(ctx); @@ -588,7 +588,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UOpsSymType **values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = _Py_uop_sym_new_unknown(ctx); @@ -599,7 +599,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UOpsSymType **values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = _Py_uop_sym_new_unknown(ctx); @@ -610,15 +610,15 @@ } case _UNPACK_EX: { - _Py_UOpsSymType *seq; - _Py_UOpsSymType **values; + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = _Py_uop_sym_new_unknown(ctx)); + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); } stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; @@ -644,7 +644,7 @@ } case _LOAD_LOCALS: { - _Py_UOpsSymType *locals; + _Py_UopsSymbol *locals; locals = _Py_uop_sym_new_unknown(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; @@ -653,7 +653,7 @@ } case _LOAD_FROM_DICT_OR_GLOBALS: { - _Py_UOpsSymType *v; + _Py_UopsSymbol *v; v = _Py_uop_sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; @@ -661,7 +661,7 @@ } case _LOAD_NAME: { - _Py_UOpsSymType *v; + _Py_UopsSymbol *v; v = _Py_uop_sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; @@ -670,8 +670,8 @@ } case _LOAD_GLOBAL: { - _Py_UOpsSymType *res; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; null = _Py_uop_sym_new_null(ctx); @@ -691,8 +691,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UOpsSymType *res; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; null = _Py_uop_sym_new_null(ctx); @@ -704,8 +704,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UOpsSymType *res; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; null = _Py_uop_sym_new_null(ctx); @@ -729,7 +729,7 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; @@ -737,7 +737,7 @@ } case _LOAD_DEREF: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = _Py_uop_sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; @@ -755,7 +755,7 @@ } case _BUILD_STRING: { - _Py_UOpsSymType *str; + _Py_UopsSymbol *str; str = _Py_uop_sym_new_unknown(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; @@ -764,7 +764,7 @@ } case _BUILD_TUPLE: { - _Py_UOpsSymType *tup; + _Py_UopsSymbol *tup; tup = _Py_uop_sym_new_unknown(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; @@ -773,7 +773,7 @@ } case _BUILD_LIST: { - _Py_UOpsSymType *list; + _Py_UopsSymbol *list; list = _Py_uop_sym_new_unknown(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; @@ -792,7 +792,7 @@ } case _BUILD_SET: { - _Py_UOpsSymType *set; + _Py_UopsSymbol *set; set = _Py_uop_sym_new_unknown(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; @@ -801,7 +801,7 @@ } case _BUILD_MAP: { - _Py_UOpsSymType *map; + _Py_UopsSymbol *map; map = _Py_uop_sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; @@ -814,7 +814,7 @@ } case _BUILD_CONST_KEY_MAP: { - _Py_UOpsSymType *map; + _Py_UopsSymbol *map; map = _Py_uop_sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; @@ -840,7 +840,7 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UOpsSymType *attr; + _Py_UopsSymbol *attr; attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; @@ -849,8 +849,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self_or_null; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; self_or_null = _Py_uop_sym_new_unknown(ctx); @@ -862,8 +862,8 @@ } case _LOAD_ATTR: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self_or_null = NULL; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null = NULL; attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; self_or_null = _Py_uop_sym_new_unknown(ctx); @@ -883,9 +883,9 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -898,12 +898,12 @@ } case _CHECK_ATTR_MODULE: { - _Py_UOpsSymType *owner; + _Py_UopsSymbol *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand; (void)dict_version; - if (_Py_uop_sym_is_const(owner)) { - PyObject *cnst = _Py_uop_sym_get_const(owner); + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); if (PyModule_CheckExact(cnst)) { PyModuleObject *mod = (PyModuleObject *)cnst; PyObject *dict = mod->md_dict; @@ -919,29 +919,29 @@ } case _LOAD_ATTR_MODULE: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. - assert(_Py_uop_sym_is_const(owner)); - PyModuleObject *mod = (PyModuleObject *)_Py_uop_sym_get_const(owner); + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_const(ctx, res)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -954,9 +954,9 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -969,9 +969,9 @@ } case _LOAD_ATTR_SLOT: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -988,9 +988,9 @@ } case _LOAD_ATTR_CLASS: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -1023,7 +1023,7 @@ } case _COMPARE_OP: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1032,7 +1032,7 @@ } case _COMPARE_OP_FLOAT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1041,7 +1041,7 @@ } case _COMPARE_OP_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1050,7 +1050,7 @@ } case _COMPARE_OP_STR: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1059,7 +1059,7 @@ } case _IS_OP: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; @@ -1068,7 +1068,7 @@ } case _CONTAINS_OP: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; @@ -1077,8 +1077,8 @@ } case _CHECK_EG_MATCH: { - _Py_UOpsSymType *rest; - _Py_UOpsSymType *match; + _Py_UopsSymbol *rest; + _Py_UopsSymbol *match; rest = _Py_uop_sym_new_unknown(ctx); if (rest == NULL) goto out_of_space; match = _Py_uop_sym_new_unknown(ctx); @@ -1089,7 +1089,7 @@ } case _CHECK_EXC_MATCH: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; @@ -1101,7 +1101,7 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = _Py_uop_sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; @@ -1109,7 +1109,7 @@ } case _GET_LEN: { - _Py_UOpsSymType *len_o; + _Py_UopsSymbol *len_o; len_o = _Py_uop_sym_new_unknown(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; @@ -1118,7 +1118,7 @@ } case _MATCH_CLASS: { - _Py_UOpsSymType *attrs; + _Py_UopsSymbol *attrs; attrs = _Py_uop_sym_new_unknown(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; @@ -1127,7 +1127,7 @@ } case _MATCH_MAPPING: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; @@ -1136,7 +1136,7 @@ } case _MATCH_SEQUENCE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; @@ -1145,7 +1145,7 @@ } case _MATCH_KEYS: { - _Py_UOpsSymType *values_or_none; + _Py_UopsSymbol *values_or_none; values_or_none = _Py_uop_sym_new_unknown(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; @@ -1154,7 +1154,7 @@ } case _GET_ITER: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -1162,7 +1162,7 @@ } case _GET_YIELD_FROM_ITER: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = _Py_uop_sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -1172,7 +1172,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UOpsSymType *next; + _Py_UopsSymbol *next; next = _Py_uop_sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; @@ -1193,7 +1193,7 @@ } case _ITER_NEXT_LIST: { - _Py_UOpsSymType *next; + _Py_UopsSymbol *next; next = _Py_uop_sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; @@ -1212,7 +1212,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UOpsSymType *next; + _Py_UopsSymbol *next; next = _Py_uop_sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; @@ -1231,10 +1231,10 @@ } case _ITER_NEXT_RANGE: { - _Py_UOpsSymType *iter; - _Py_UOpsSymType *next; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *next; iter = stack_pointer[-1]; - OUT_OF_SPACE_IF_NULL(next = _Py_uop_sym_new_type(ctx, &PyLong_Type)); + OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); (void)iter; stack_pointer[0] = next; stack_pointer += 1; @@ -1244,8 +1244,8 @@ /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ case _BEFORE_ASYNC_WITH: { - _Py_UOpsSymType *exit; - _Py_UOpsSymType *res; + _Py_UopsSymbol *exit; + _Py_UopsSymbol *res; exit = _Py_uop_sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; res = _Py_uop_sym_new_unknown(ctx); @@ -1257,8 +1257,8 @@ } case _BEFORE_WITH: { - _Py_UOpsSymType *exit; - _Py_UOpsSymType *res; + _Py_UopsSymbol *exit; + _Py_UopsSymbol *res; exit = _Py_uop_sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; res = _Py_uop_sym_new_unknown(ctx); @@ -1270,7 +1270,7 @@ } case _WITH_EXCEPT_START: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; @@ -1279,8 +1279,8 @@ } case _PUSH_EXC_INFO: { - _Py_UOpsSymType *prev_exc; - _Py_UOpsSymType *new_exc; + _Py_UopsSymbol *prev_exc; + _Py_UopsSymbol *new_exc; prev_exc = _Py_uop_sym_new_unknown(ctx); if (prev_exc == NULL) goto out_of_space; new_exc = _Py_uop_sym_new_unknown(ctx); @@ -1300,13 +1300,13 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1315,13 +1315,13 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1330,7 +1330,7 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UOpsSymType *attr; + _Py_UopsSymbol *attr; attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; @@ -1338,7 +1338,7 @@ } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UOpsSymType *attr; + _Py_UopsSymbol *attr; attr = _Py_uop_sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; @@ -1350,13 +1350,13 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1369,23 +1369,23 @@ /* _CALL is not a viable micro-op for tier 2 */ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UOpsSymType *null; - _Py_UOpsSymType *callable; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - _Py_uop_sym_set_null(null); - _Py_uop_sym_set_type(callable, &PyMethod_Type); + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UOpsSymType *callable; - _Py_UOpsSymType *func; - _Py_UOpsSymType *self; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *self; callable = stack_pointer[-2 - oparg]; (void)callable; - OUT_OF_SPACE_IF_NULL(func = _Py_uop_sym_new_not_null(ctx)); - OUT_OF_SPACE_IF_NULL(self = _Py_uop_sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -1396,12 +1396,12 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UOpsSymType *self_or_null; - _Py_UOpsSymType *callable; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand; - _Py_uop_sym_set_type(callable, &PyFunction_Type); + sym_set_type(callable, &PyFunction_Type); (void)self_or_null; (void)func_version; break; @@ -1412,9 +1412,9 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UOpsSymType **args; - _Py_UOpsSymType *self_or_null; - _Py_UOpsSymType *callable; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -1428,23 +1428,23 @@ PyCodeObject *co = (PyCodeObject *)func->func_code; assert(self_or_null != NULL); assert(args != NULL); - if (_Py_uop_sym_is_not_null(self_or_null)) { + if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - _Py_UOpsSymType **localsplus_start = ctx->n_consumed; + _Py_UopsSymbol **localsplus_start = ctx->n_consumed; int n_locals_already_filled = 0; // Can determine statically, so we interleave the new locals // and make the current stack the new locals. // This also sets up for true call inlining. - if (_Py_uop_sym_is_null(self_or_null) || _Py_uop_sym_is_not_null(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; } OUT_OF_SPACE_IF_NULL(new_frame = - _Py_uop_ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); - stack_pointer[-2 - oparg] = (_Py_UOpsSymType *)new_frame; + frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; } @@ -1463,7 +1463,7 @@ /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ case _CALL_TYPE_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1472,7 +1472,7 @@ } case _CALL_STR_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1481,7 +1481,7 @@ } case _CALL_TUPLE_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1497,7 +1497,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1506,7 +1506,7 @@ } case _CALL_BUILTIN_O: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1515,7 +1515,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1524,7 +1524,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1533,7 +1533,7 @@ } case _CALL_LEN: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1542,7 +1542,7 @@ } case _CALL_ISINSTANCE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1551,7 +1551,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1560,7 +1560,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1569,7 +1569,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1578,7 +1578,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1595,7 +1595,7 @@ /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UOpsSymType *func; + _Py_UopsSymbol *func; func = _Py_uop_sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; @@ -1603,7 +1603,7 @@ } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UOpsSymType *func; + _Py_UopsSymbol *func; func = _Py_uop_sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; @@ -1612,7 +1612,7 @@ } case _BUILD_SLICE: { - _Py_UOpsSymType *slice; + _Py_UopsSymbol *slice; slice = _Py_uop_sym_new_unknown(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; @@ -1621,7 +1621,7 @@ } case _CONVERT_VALUE: { - _Py_UOpsSymType *result; + _Py_UopsSymbol *result; result = _Py_uop_sym_new_unknown(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; @@ -1629,7 +1629,7 @@ } case _FORMAT_SIMPLE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -1637,7 +1637,7 @@ } case _FORMAT_WITH_SPEC: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1646,8 +1646,8 @@ } case _COPY: { - _Py_UOpsSymType *bottom; - _Py_UOpsSymType *top; + _Py_UopsSymbol *bottom; + _Py_UopsSymbol *top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -1657,7 +1657,7 @@ } case _BINARY_OP: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1666,8 +1666,8 @@ } case _SWAP: { - _Py_UOpsSymType *top; - _Py_UOpsSymType *bottom; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; stack_pointer[-2 - (oparg-2)] = top; @@ -1730,29 +1730,29 @@ } case _LOAD_CONST_INLINE: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; break; } case _LOAD_CONST_INLINE_BORROW: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UOpsSymType *value; - _Py_UOpsSymType *null; + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -1760,11 +1760,11 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UOpsSymType *value; - _Py_UOpsSymType *null; + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = _Py_uop_sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 1b7c88cf785fd2..794d73733f85a7 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -32,10 +32,10 @@ static inline int get_lltrace(void) { #endif // Takes a borrowed reference to const_val, turns that into a strong reference. -static _Py_UOpsSymType* -sym_new(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) +static _Py_UopsSymbol * +sym_new(_Py_UOpsContext *ctx, PyObject *const_val) { - _Py_UOpsSymType *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { OPT_STAT_INC(optimizer_failure_reason_no_memory); DPRINTF(1, "out of space for symbolic expression type\n"); @@ -54,37 +54,37 @@ sym_new(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) } static inline void -sym_set_flag(_Py_UOpsSymType *sym, int flag) +sym_set_flag(_Py_UopsSymbol *sym, int flag) { sym->flags |= flag; } static inline bool -sym_has_flag(_Py_UOpsSymType *sym, int flag) +sym_has_flag(_Py_UopsSymbol *sym, int flag) { return (sym->flags & flag) != 0; } bool -_Py_uop_sym_is_not_null(_Py_UOpsSymType *sym) +_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) { return (sym->flags & (IS_NULL | NOT_NULL)) == NOT_NULL; } bool -_Py_uop_sym_is_null(_Py_UOpsSymType *sym) +_Py_uop_sym_is_null(_Py_UopsSymbol *sym) { return (sym->flags & (IS_NULL | NOT_NULL)) == IS_NULL; } bool -_Py_uop_sym_is_const(_Py_UOpsSymType *sym) +_Py_uop_sym_is_const(_Py_UopsSymbol *sym) { return (sym->flags & TRUE_CONST) != 0; } PyObject * -_Py_uop_sym_get_const(_Py_UOpsSymType *sym) +_Py_uop_sym_get_const(_Py_UopsSymbol *sym) { assert(_Py_uop_sym_is_const(sym)); assert(sym->const_val); @@ -92,7 +92,7 @@ _Py_uop_sym_get_const(_Py_UOpsSymType *sym) } void -_Py_uop_sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp) +_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *tp) { assert(PyType_Check(tp)); sym->typ = tp; @@ -101,23 +101,23 @@ _Py_uop_sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp) } void -_Py_uop_sym_set_null(_Py_UOpsSymType *sym) +_Py_uop_sym_set_null(_Py_UopsSymbol *sym) { sym_set_flag(sym, IS_NULL); sym_set_flag(sym, KNOWN); } -_Py_UOpsSymType * -_Py_uop_sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx) +_Py_UopsSymbol * +_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) { - return sym_new(ctx,NULL); + return sym_new(ctx, NULL); } -_Py_UOpsSymType * -_Py_uop_sym_new_not_null(_Py_UOpsAbstractInterpContext *ctx) +_Py_UopsSymbol * +_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) { - _Py_UOpsSymType *res = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) { return NULL; } @@ -126,11 +126,11 @@ _Py_uop_sym_new_not_null(_Py_UOpsAbstractInterpContext *ctx) return res; } -_Py_UOpsSymType * -_Py_uop_sym_new_type(_Py_UOpsAbstractInterpContext *ctx, +_Py_UopsSymbol * +_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) { - _Py_UOpsSymType *res = sym_new(ctx,NULL); + _Py_UopsSymbol *res = sym_new(ctx,NULL); if (res == NULL) { return NULL; } @@ -139,14 +139,11 @@ _Py_uop_sym_new_type(_Py_UOpsAbstractInterpContext *ctx, } // Takes a borrowed reference to const_val. -_Py_UOpsSymType* -_Py_uop_sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) +_Py_UopsSymbol * +_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) { assert(const_val != NULL); - _Py_UOpsSymType *temp = sym_new( - ctx, - const_val - ); + _Py_UopsSymbol *temp = sym_new(ctx, const_val); if (temp == NULL) { return NULL; } @@ -157,10 +154,10 @@ _Py_uop_sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) return temp; } -_Py_UOpsSymType* -_Py_uop_sym_new_null(_Py_UOpsAbstractInterpContext *ctx) +_Py_UopsSymbol * +_Py_uop_sym_new_null(_Py_UOpsContext *ctx) { - _Py_UOpsSymType *null_sym = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); if (null_sym == NULL) { return NULL; } @@ -169,7 +166,7 @@ _Py_uop_sym_new_null(_Py_UOpsAbstractInterpContext *ctx) } bool -_Py_uop_sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { assert(typ == NULL || PyType_Check(typ)); if (!sym_has_flag(sym, KNOWN)) { @@ -180,13 +177,12 @@ _Py_uop_sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ) // 0 on success, -1 on error. _Py_UOpsAbstractFrame * -_Py_uop_ctx_frame_new( - _Py_UOpsAbstractInterpContext *ctx, +_Py_uop_frame_new( + _Py_UOpsContext *ctx, PyCodeObject *co, - _Py_UOpsSymType **localsplus_start, + _Py_UopsSymbol **localsplus_start, int n_locals_already_filled, - int curr_stackentries -) + int curr_stackentries) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; @@ -205,7 +201,7 @@ _Py_uop_ctx_frame_new( // Initialize with the initial state of all local variables for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { - _Py_UOpsSymType *local = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); if (local == NULL) { return NULL; } @@ -215,7 +211,7 @@ _Py_uop_ctx_frame_new( // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - _Py_UOpsSymType *stackvar = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); if (stackvar == NULL) { return NULL; } @@ -226,7 +222,7 @@ _Py_uop_ctx_frame_new( } void -_Py_uop_abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx) +_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) { if (ctx == NULL) { return; @@ -239,9 +235,7 @@ _Py_uop_abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx) } int -_Py_uop_abstractcontext_init( - _Py_UOpsAbstractInterpContext *ctx -) +_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) { ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; @@ -262,12 +256,9 @@ _Py_uop_abstractcontext_init( } int -_Py_uop_ctx_frame_pop( - _Py_UOpsAbstractInterpContext *ctx -) +_Py_uop_frame_pop(_Py_UOpsContext *ctx) { _Py_UOpsAbstractFrame *frame = ctx->frame; - ctx->n_consumed = frame->locals; ctx->curr_frame_depth--; assert(ctx->curr_frame_depth >= 1); @@ -287,10 +278,10 @@ do { \ } while (0) /* -static _Py_UOpsSymType * -make_bottom(_Py_UOpsAbstractInterpContext *ctx) +static _Py_UopsSymbol * +make_contradiction(_Py_UOpsContext *ctx) { - _Py_UOpsSymType *sym = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); _Py_uop_sym_set_null(sym); _Py_uop_sym_set_type(sym, &PyLong_Type); return sym; @@ -299,11 +290,11 @@ make_bottom(_Py_UOpsAbstractInterpContext *ctx) PyObject * _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { - _Py_UOpsAbstractInterpContext context; - _Py_UOpsAbstractInterpContext *ctx = &context; + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; _Py_uop_abstractcontext_init(ctx); - _Py_UOpsSymType *top = _Py_uop_sym_new_unknown(ctx); + _Py_UopsSymbol *top = _Py_uop_sym_new_unknown(ctx); if (top == NULL) { return NULL; } @@ -312,17 +303,17 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_const(top), "unknown is a constant"); // TEST_PREDICATE(_Py_uop_sym_get_const(top) == NULL, "unknown as constant is not NULL"); - // _Py_UOpsSymType *bottom = make_bottom(ctx); - // TEST_PREDICATE(_Py_uop_sym_is_null(bottom), "bottom is NULL is not true"); - // TEST_PREDICATE(_Py_uop_sym_is_not_null(bottom), "bottom is not NULL is not true"); - // TEST_PREDICATE(_Py_uop_sym_is_const(bottom), "bottom is a constant is not true"); + // _Py_UopsSymbol *contradiction = make_contradiction(ctx); + // TEST_PREDICATE(_Py_uop_sym_is_null(contradiction), "contradiction is NULL is not true"); + // TEST_PREDICATE(_Py_uop_sym_is_not_null(contradiction), "contradiction is not NULL is not true"); + // TEST_PREDICATE(_Py_uop_sym_is_const(contradiction), "contradiction is a constant is not true"); - _Py_UOpsSymType *int_type = _Py_uop_sym_new_type(ctx, &PyLong_Type); + _Py_UopsSymbol *int_type = _Py_uop_sym_new_type(ctx, &PyLong_Type); TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "inconsistent type"); _Py_uop_sym_set_type(int_type, &PyLong_Type); TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "inconsistent type"); _Py_uop_sym_set_type(int_type, &PyFloat_Type); - // TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "bottom doesn't match int"); + // TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "(int and float) doesn't match int"); _Py_uop_abstractcontext_fini(ctx); Py_RETURN_NONE; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index f110a74e3ad351..e933fee8f4242b 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -41,10 +41,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return f"_Py_UOpsSymType **" + return f"_Py_UopsSymbol **" if var.type: return var.type - return f"_Py_UOpsSymType *" + return f"_Py_UopsSymbol *" def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -148,7 +148,7 @@ def write_uop( if not var.peek or is_override: out.emit(stack.push(var)) out.start_line() - stack.flush(out, cast_type="_Py_UOpsSymType *") + stack.flush(out, cast_type="_Py_UopsSymbol *") except SizeMismatch as ex: raise analysis_error(ex.args[0], uop.body[0]) From 3a72fc36f93d40048371b789e32eefc97b6ade63 Mon Sep 17 00:00:00 2001 From: Tahoma Software Date: Tue, 27 Feb 2024 08:33:05 -0500 Subject: [PATCH 21/82] gh-115315: Update time.rst to include microseconds field (%f) in chart (#115316) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/time.rst | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 2782a961363666..88c15f19bedf8c 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -433,6 +433,10 @@ Functions | ``%d`` | Day of the month as a decimal number [01,31]. | | | | | | +-----------+------------------------------------------------+-------+ + | ``%f`` | Microseconds as a decimal number | \(1) | + | | [000000,999999]. | | + | | | | + +-----------+------------------------------------------------+-------+ | ``%H`` | Hour (24-hour clock) as a decimal number | | | | [00,23]. | | +-----------+------------------------------------------------+-------+ @@ -448,13 +452,13 @@ Functions | ``%M`` | Minute as a decimal number [00,59]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%p`` | Locale's equivalent of either AM or PM. | \(1) | + | ``%p`` | Locale's equivalent of either AM or PM. | \(2) | | | | | +-----------+------------------------------------------------+-------+ - | ``%S`` | Second as a decimal number [00,61]. | \(2) | + | ``%S`` | Second as a decimal number [00,61]. | \(3) | | | | | +-----------+------------------------------------------------+-------+ - | ``%U`` | Week number of the year (Sunday as the first | \(3) | + | ``%U`` | Week number of the year (Sunday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Sunday are considered to be in week 0. | | @@ -465,7 +469,7 @@ Functions | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%W`` | Week number of the year (Monday as the first | \(3) | + | ``%W`` | Week number of the year (Monday as the first | \(4) | | | day of the week) as a decimal number [00,53]. | | | | All days in a new year preceding the first | | | | Monday are considered to be in week 0. | | @@ -500,17 +504,23 @@ Functions Notes: (1) + The ``%f`` format directive only applies to :func:`strptime`, + not to :func:`strftime`. However, see also :meth:`datetime.datetime.strptime` and + :meth:`datetime.datetime.strftime` where the ``%f`` format directive + :ref:`applies to microseconds `. + + (2) When used with the :func:`strptime` function, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. .. _leap-second: - (2) + (3) The range really is ``0`` to ``61``; value ``60`` is valid in timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. - (3) + (4) When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in calculations when the day of the week and the year are specified. From 686ec17f506cddd0b14a8aad5849c15ffc20ed46 Mon Sep 17 00:00:00 2001 From: Miguel Brito <5544985+miguendes@users.noreply.github.com> Date: Tue, 27 Feb 2024 14:57:59 +0000 Subject: [PATCH 22/82] bpo-43952: Fix multiprocessing Listener authkey bug (GH-25845) Listener.accept() no longer hangs when authkey is an empty bytes object. --- Lib/multiprocessing/connection.py | 3 ++- Lib/test/_test_multiprocessing.py | 19 +++++++++++++++++++ .../2021-05-03-11-04-12.bpo-43952.Me7fJe.rst | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 58d697fdecacc0..b7e1e132172d02 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -476,8 +476,9 @@ def accept(self): ''' if self._listener is None: raise OSError('listener is closed') + c = self._listener.accept() - if self._authkey: + if self._authkey is not None: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f70a693e641b4e..058537bab5af26 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3504,6 +3504,25 @@ def test_context(self): if self.TYPE == 'processes': self.assertRaises(OSError, l.accept) + def test_empty_authkey(self): + # bpo-43952: allow empty bytes as authkey + def handler(*args): + raise RuntimeError('Connection took too long...') + + def run(addr, authkey): + client = self.connection.Client(addr, authkey=authkey) + client.send(1729) + + key = b"" + + with self.connection.Listener(authkey=key) as listener: + threading.Thread(target=run, args=(listener.address, key)).start() + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + + if self.TYPE == 'processes': + self.assertRaises(OSError, listener.accept) + @unittest.skipUnless(util.abstract_sockets_supported, "test needs abstract socket support") def test_abstract_socket(self): diff --git a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst new file mode 100644 index 00000000000000..e164619e44a301 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst @@ -0,0 +1,2 @@ +Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty bytes +as authkey. Not accepting empty bytes as key causes it to hang indefinitely. From a355f60b032306651ca27bc53bbb82eb5106ff71 Mon Sep 17 00:00:00 2001 From: "Pierre Ossman (ThinLinc team)" Date: Wed, 28 Feb 2024 02:27:44 +0100 Subject: [PATCH 23/82] gh-114914: Avoid keeping dead StreamWriter alive (#115661) In some cases we might cause a StreamWriter to stay alive even when the application has dropped all references to it. This prevents us from doing automatical cleanup, and complaining that the StreamWriter wasn't properly closed. Fortunately, the extra reference was never actually used for anything so we can just drop it. --- Lib/asyncio/streams.py | 14 +++-------- Lib/test/test_asyncio/test_streams.py | 25 +++++++++++++++++++ ...-02-19-15-52-30.gh-issue-114914.M5-1d8.rst | 2 ++ 3 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index df58b7a799a5ad..3fe52dbac25c91 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None): # is established. self._strong_reader = stream_reader self._reject_connection = False - self._stream_writer = None self._task = None self._transport = None self._client_connected_cb = client_connected_cb @@ -214,10 +213,8 @@ def _stream_reader(self): return None return self._stream_reader_wr() - def _replace_writer(self, writer): + def _replace_transport(self, transport): loop = self._loop - transport = writer.transport - self._stream_writer = writer self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -239,11 +236,8 @@ def connection_made(self, transport): reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - reader, - self._loop) - res = self._client_connected_cb(reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): def callback(task): if task.cancelled(): @@ -405,7 +399,7 @@ async def start_tls(self, sslcontext, *, ssl_handshake_timeout=ssl_handshake_timeout, ssl_shutdown_timeout=ssl_shutdown_timeout) self._transport = new_transport - protocol._replace_writer(self) + protocol._replace_transport(new_transport) def __del__(self, warnings=warnings): if not self._transport.is_closing(): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 210990593adfa9..bf123ebf9bd158 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1130,6 +1130,31 @@ async def inner(httpd): self.assertEqual(messages, []) + def test_unclosed_server_resource_warnings(self): + async def inner(rd, wr): + fut.set_result(True) + with self.assertWarns(ResourceWarning) as cm: + del wr + gc.collect() + self.assertEqual(len(cm.warnings), 1) + self.assertTrue(str(cm.warnings[0].message).startswith("unclosed Date: Wed, 28 Feb 2024 02:39:08 +0100 Subject: [PATCH 24/82] gh-112997: Don't log arguments in asyncio unless debugging (#115667) Nothing else in Python generally logs the contents of variables, so this can be very unexpected for developers and could leak sensitive information in to terminals and log files. --- Lib/asyncio/events.py | 6 +++-- Lib/asyncio/format_helpers.py | 22 +++++++++++------ Lib/test/test_asyncio/test_events.py | 24 ++++++++++++++++--- ...-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst | 2 ++ 4 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 072a99fee123c3..680749325025db 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -54,7 +54,8 @@ def _repr_info(self): info.append('cancelled') if self._callback is not None: info.append(format_helpers._format_callback_source( - self._callback, self._args)) + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -90,7 +91,8 @@ def _run(self): raise except BaseException as exc: cb = format_helpers._format_callback_source( - self._callback, self._args) + self._callback, self._args, + debug=self._loop.get_debug()) msg = f'Exception in callback {cb}' context = { 'message': msg, diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 27d11fd4fa9553..93737b7708a67b 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -19,19 +19,26 @@ def _get_function_source(func): return None -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) source = _get_function_source(func) if source: func_repr += f' at {source[0]}:{source[1]}' return func_repr -def _format_args_and_kwargs(args, kwargs): +def _format_args_and_kwargs(args, kwargs, *, debug=False): """Format function arguments and keyword arguments. Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. """ + if not debug: + return '()' + # use reprlib to limit the length of the output items = [] if args: @@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs): return '({})'.format(', '.join(items)) -def _format_callback(func, args, kwargs, suffix=''): +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) if hasattr(func, '__qualname__') and func.__qualname__: func_repr = func.__qualname__ @@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''): else: func_repr = repr(func) - func_repr += _format_args_and_kwargs(args, kwargs) + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) if suffix: func_repr += suffix return func_repr diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index c92c88bd5b2429..5b9c871e1d1b5a 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2250,7 +2250,7 @@ def test_handle_repr(self): h = asyncio.Handle(noop, (1, 2), self.loop) filename, lineno = test_utils.get_function_source(noop) self.assertEqual(repr(h), - '' + '' % (filename, lineno)) # cancelled handle @@ -2268,14 +2268,14 @@ def test_handle_repr(self): # partial function cb = functools.partial(noop, 1, 2) h = asyncio.Handle(cb, (3,), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) # partial function with keyword args cb = functools.partial(noop, x=1) h = asyncio.Handle(cb, (2, 3), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) @@ -2316,6 +2316,24 @@ def test_handle_repr_debug(self): '' % (filename, lineno, create_filename, create_lineno)) + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + def test_handle_source_traceback(self): loop = asyncio.get_event_loop_policy().new_event_loop() loop.set_debug(True) diff --git a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst new file mode 100644 index 00000000000000..4f97b2d6085ba0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst @@ -0,0 +1,2 @@ +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. From ed4dfd8825b49e16a0fcb9e67baf1b58bb8d438f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 27 Feb 2024 18:13:03 -0800 Subject: [PATCH 25/82] gh-105858: Improve AST node constructors (#105880) Demonstration: >>> ast.FunctionDef.__annotations__ {'name': , 'args': , 'body': list[ast.stmt], 'decorator_list': list[ast.expr], 'returns': ast.expr | None, 'type_comment': str | None, 'type_params': list[ast.type_param]} >>> ast.FunctionDef() :1: DeprecationWarning: FunctionDef.__init__ missing 1 required positional argument: 'name'. This will become an error in Python 3.15. :1: DeprecationWarning: FunctionDef.__init__ missing 1 required positional argument: 'args'. This will become an error in Python 3.15. >>> node = ast.FunctionDef(name="foo", args=ast.arguments()) >>> node.decorator_list [] >>> ast.FunctionDef(whatever="you want", name="x", args=ast.arguments()) :1: DeprecationWarning: FunctionDef.__init__ got an unexpected keyword argument 'whatever'. Support for arbitrary keyword arguments is deprecated and will be removed in Python 3.15. --- Doc/library/ast.rst | 25 +- Doc/whatsnew/3.13.rst | 15 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_ast.py | 101 +- ...-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst | 8 + Parser/asdl_c.py | 238 +- Python/Python-ast.c | 4333 ++++++++++++++++- 10 files changed, 4676 insertions(+), 50 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index c943c2f498173e..d629393f023c8a 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -103,20 +103,15 @@ Node classes For example, to create and populate an :class:`ast.UnaryOp` node, you could use :: - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) + If a field that is optional in the grammar is omitted from the constructor, + it defaults to ``None``. If a list field is omitted, it defaults to the empty + list. If any other field is omitted, a :exc:`DeprecationWarning` is raised + and the AST node will not have this field. In Python 3.15, this condition will + raise an error. + .. versionchanged:: 3.8 Class :class:`ast.Constant` is now used for all constants. @@ -140,6 +135,14 @@ Node classes In the meantime, instantiating them will return an instance of a different class. +.. deprecated-removed:: 3.13 3.15 + + Previous versions of Python allowed the creation of AST nodes that were missing + required fields. Similarly, AST node constructors allowed arbitrary keyword + arguments that were set as attributes of the AST node, even if they did not + match any of the fields of the AST node. This behavior is deprecated and will + be removed in Python 3.15. + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ca5a0e7515994f..3a277d7ce1585f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -206,6 +206,21 @@ array ast --- +* The constructors of node types in the :mod:`ast` module are now stricter + in the arguments they accept, and have more intuitive behaviour when + arguments are omitted. + + If an optional field on an AST node is not included as an argument when + constructing an instance, the field will now be set to ``None``. Similarly, + if a list field is omitted, that field will now be set to an empty list. + (Previously, in both cases, the attribute would be missing on the newly + constructed AST node instance.) + + If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. + This will cause an exception in Python 3.15. Similarly, passing a keyword + argument that does not map to a field on the AST node is now deprecated, + and will raise an exception in Python 3.15. + * :func:`ast.parse` now accepts an optional argument ``optimize`` which is passed on to the :func:`compile` built-in. This makes it possible to obtain an optimized ``AST``. diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 3253b5271a9b7c..e8b62bd0dcd369 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -753,6 +753,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_dealloc_warn)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_feature_version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_field_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fields_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8780f7ef949165..b41f5c8952021e 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -242,6 +242,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_check_retval_) STRUCT_FOR_ID(_dealloc_warn) STRUCT_FOR_ID(_feature_version) + STRUCT_FOR_ID(_field_types) STRUCT_FOR_ID(_fields_) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index a9d514856dab1f..016eae02a2103d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -751,6 +751,7 @@ extern "C" { INIT_ID(_check_retval_), \ INIT_ID(_dealloc_warn), \ INIT_ID(_feature_version), \ + INIT_ID(_field_types), \ INIT_ID(_fields_), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f3b064e2a2cb25..64c4cf8c077056 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -567,6 +567,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_feature_version); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_field_types); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_fields_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d49c149da42592..7cecf319e3638f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -525,17 +525,38 @@ def test_field_attr_existence(self): if name == 'Index': continue if self._is_ast_node(name, item): - x = item() + x = self._construct_ast_class(item) if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = 'capybara' + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b'capybara' + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + def test_arguments(self): x = ast.arguments() self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args + self.assertEqual(x.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) + + self.assertEqual(x.args, []) self.assertIsNone(x.vararg) x = ast.arguments(*range(1, 8)) @@ -551,7 +572,7 @@ def test_field_attr_writable_deprecated(self): self.assertEqual(x._fields, 666) def test_field_attr_writable(self): - x = ast.Constant() + x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) @@ -611,15 +632,22 @@ def test_classattrs_deprecated(self): self.assertEqual([str(w.message) for w in wlog], [ 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + 'an error in Python 3.15.', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + 'an error in Python 3.15.', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " + 'arbitrary keyword arguments is deprecated and will be removed in Python ' + '3.15.', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', @@ -636,7 +664,8 @@ def test_classattrs_deprecated(self): ]) def test_classattrs(self): - x = ast.Constant() + with self.assertWarns(DeprecationWarning): + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): @@ -651,7 +680,7 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Constant(lineno=2) + x = ast.Constant(lineno=2, value=3) self.assertEqual(x.lineno, 2) x = ast.Constant(42, lineno=0) @@ -662,8 +691,9 @@ def test_classattrs(self): self.assertRaises(TypeError, ast.Constant, 1, None, 2) self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) @@ -815,11 +845,11 @@ def test_isinstance(self): assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) class S(str): pass with assertStrDeprecated(): @@ -888,8 +918,9 @@ def test_module(self): self.assertEqual(x.body, body) def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too @@ -927,8 +958,9 @@ def test_nodeclasses(self): self.assertEqual(x.right, 3) self.assertEqual(x.lineno, 0) - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) def test_no_fields(self): @@ -941,8 +973,9 @@ def test_pickling(self): for protocol in range(pickle.HIGHEST_PROTOCOL + 1): for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) + with self.subTest(ast=ast, protocol=protocol): + ast2 = pickle.loads(pickle.dumps(ast, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(ast)) def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) @@ -1310,8 +1343,9 @@ def test_copy_location(self): 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' 'col_offset=0, end_lineno=1, end_col_offset=5))' ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) @@ -1570,15 +1604,15 @@ def test_level_as_none(self): self.assertIn('sleep', ns) def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e with self.assertRaises(RecursionError): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = f f.operand = e with self.assertRaises(RecursionError): @@ -2866,6 +2900,23 @@ def visit_Call(self, node: ast.Call): self.assertASTTransformation(PrintToLog, code, expected) +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): + node = ast.FunctionDef(args=args) + self.assertFalse(hasattr(node, "name")) + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') + self.assertEqual(node.decorator_list, []) + + @support.cpython_only class ModuleStateTests(unittest.TestCase): # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst new file mode 100644 index 00000000000000..9338f662374c9a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst @@ -0,0 +1,8 @@ +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to ``None``. +AST nodes now have an +``__annotations__`` attribute with the expected types of their attributes. +Passing unrecognized extra arguments to AST nodes is deprecated and will +become an error in Python 3.15. Omitting a required argument to an AST node +is deprecated and will become an error in Python 3.15. Patch by Jelle +Zijlstra. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ce92672bf00776..865fd76acf697d 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -15,6 +15,13 @@ MAX_COL = 80 AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" +builtin_type_to_c_type = { + "identifier": "PyUnicode_Type", + "string": "PyUnicode_Type", + "int": "PyLong_Type", + "constant": "PyBaseObject_Type", +} + def get_c_type(name): """Return a string for the C name of the type. @@ -764,6 +771,67 @@ def visitConstructor(self, cons, name): self.emit("};",0) +class AnnotationsVisitor(PickleVisitor): + def visitModule(self, mod): + self.file.write(textwrap.dedent(''' + static int + add_ast_annotations(struct ast_state *state) + { + bool cond; + ''')) + for dfn in mod.dfns: + self.visit(dfn) + self.file.write(textwrap.dedent(''' + return 1; + } + ''')) + + def visitProduct(self, prod, name): + self.emit_annotations(name, prod.fields) + + def visitSum(self, sum, name): + for t in sum.types: + self.visitConstructor(t, name) + + def visitConstructor(self, cons, name): + self.emit_annotations(cons.name, cons.fields) + + def emit_annotations(self, name, fields): + self.emit(f"PyObject *{name}_annotations = PyDict_New();", 1) + self.emit(f"if (!{name}_annotations) return 0;", 1) + for field in fields: + self.emit("{", 1) + if field.type in builtin_type_to_c_type: + self.emit(f"PyObject *type = (PyObject *)&{builtin_type_to_c_type[field.type]};", 2) + else: + self.emit(f"PyObject *type = state->{field.type}_type;", 2) + if field.opt: + self.emit("type = _Py_union_type_or(type, Py_None);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + elif field.seq: + self.emit("type = Py_GenericAlias((PyObject *)&PyList_Type, type);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + else: + self.emit("Py_INCREF(type);", 2) + self.emit(f"cond = PyDict_SetItemString({name}_annotations, \"{field.name}\", type) == 0;", 2) + self.emit("Py_DECREF(type);", 2) + self.emit_annotations_error(name, 2) + self.emit("}", 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "_field_types", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "__annotations__", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f"Py_DECREF({name}_annotations);", 1) + + def emit_annotations_error(self, name, depth): + self.emit("if (!cond) {", depth) + self.emit(f"Py_DECREF({name}_annotations);", depth + 1) + self.emit("return 0;", depth + 1) + self.emit("}", depth) + + class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): @@ -812,7 +880,7 @@ def visitModule(self, mod): Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -821,6 +889,13 @@ def visitModule(self, mod): if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -840,6 +915,11 @@ def visitModule(self, mod): goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -852,13 +932,14 @@ def visitModule(self, mod): if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -866,15 +947,91 @@ def visitModule(self, mod): goto cleanup; } } + else if ( + PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 + ) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types)) { + res = -1; + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, name); + } + goto set_remaining_cleanup; + } + if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + res = -1; + goto cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -886,14 +1043,75 @@ def visitModule(self, mod): return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL, + *remaining_dict = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + // Serialize the fields as positional args if possible, because if we + // serialize them as a dict, during unpickling they are set only *after* + // the object is constructed, which will now trigger a DeprecationWarning + // if the AST type has required fields. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + remaining_dict = PyDict_Copy(dict); + Py_DECREF(dict); + if (!remaining_dict) { + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value = PyDict_GetItemWithError(remaining_dict, name); + if (!value) { + if (PyErr_Occurred()) { + goto cleanup; + } + break; + } + if (PyList_Append(positional_args, value) < 0) { + goto cleanup; + } + if (PyDict_DelItem(remaining_dict, name) < 0) { + goto cleanup; + } + Py_DECREF(name); + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple, + remaining_dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); } - return Py_BuildValue("O()", Py_TYPE(self)); +cleanup: + Py_XDECREF(fields); + Py_XDECREF(remaining_fields); + Py_XDECREF(remaining_dict); + Py_XDECREF(positional_args); + return result; } static PyMemberDef ast_type_members[] = { @@ -1117,6 +1335,9 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' + if (!add_ast_annotations(state)) { + return -1; + } return 0; } ''')) @@ -1534,6 +1755,8 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() + #include "pycore_unionobject.h" // _Py_union_type_or + #include "structmember.h" #include struct validator { @@ -1651,6 +1874,7 @@ def write_source(mod, metadata, f, internal_h_file): v = ChainOfVisitors( SequenceConstructorVisitor(f), PyTypesDeclareVisitor(f), + AnnotationsVisitor(f), PyTypesVisitor(f), Obj2ModPrototypeVisitor(f), FunctionVisitor(f), diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d77e986ba067a3..46387493214829 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -7,6 +7,8 @@ #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_unionobject.h" // _Py_union_type_or +#include "structmember.h" #include struct validator { @@ -816,6 +818,4170 @@ static const char * const TypeVarTuple_fields[]={ }; +static int +add_ast_annotations(struct ast_state *state) +{ + bool cond; + PyObject *Module_annotations = PyDict_New(); + if (!Module_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + { + PyObject *type = state->type_ignore_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "type_ignores", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Module_type, "_field_types", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Module_type, "__annotations__", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + Py_DECREF(Module_annotations); + PyObject *Interactive_annotations = PyDict_New(); + if (!Interactive_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyDict_SetItemString(Interactive_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Interactive_type, "_field_types", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Interactive_type, "__annotations__", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + Py_DECREF(Interactive_annotations); + PyObject *Expression_annotations = PyDict_New(); + if (!Expression_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expression_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expression_type, "_field_types", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expression_type, "__annotations__", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + Py_DECREF(Expression_annotations); + PyObject *FunctionType_annotations = PyDict_New(); + if (!FunctionType_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionType_annotations, "argtypes", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionType_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionType_type, "_field_types", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionType_type, "__annotations__", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + Py_DECREF(FunctionType_annotations); + PyObject *FunctionDef_annotations = PyDict_New(); + if (!FunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_params", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "_field_types", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "__annotations__", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + Py_DECREF(FunctionDef_annotations); + PyObject *AsyncFunctionDef_annotations = PyDict_New(); + if (!AsyncFunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "name", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "args", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "body", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "decorator_list", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "returns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_params", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, "_field_types", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, + "__annotations__", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + Py_DECREF(AsyncFunctionDef_annotations); + PyObject *ClassDef_annotations = PyDict_New(); + if (!ClassDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ClassDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "bases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "keywords", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ClassDef_type, "_field_types", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ClassDef_type, "__annotations__", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + Py_DECREF(ClassDef_annotations); + PyObject *Return_annotations = PyDict_New(); + if (!Return_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyDict_SetItemString(Return_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Return_type, "_field_types", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Return_type, "__annotations__", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + Py_DECREF(Return_annotations); + PyObject *Delete_annotations = PyDict_New(); + if (!Delete_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyDict_SetItemString(Delete_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Delete_type, "_field_types", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Delete_type, "__annotations__", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + Py_DECREF(Delete_annotations); + PyObject *Assign_annotations = PyDict_New(); + if (!Assign_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assign_type, "_field_types", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assign_type, "__annotations__", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + Py_DECREF(Assign_annotations); + PyObject *TypeAlias_annotations = PyDict_New(); + if (!TypeAlias_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeAlias_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "_field_types", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "__annotations__", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + Py_DECREF(TypeAlias_annotations); + PyObject *AugAssign_annotations = PyDict_New(); + if (!AugAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AugAssign_type, "_field_types", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AugAssign_type, "__annotations__", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + Py_DECREF(AugAssign_annotations); + PyObject *AnnAssign_annotations = PyDict_New(); + if (!AnnAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "annotation", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyDict_SetItemString(AnnAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "simple", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "_field_types", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "__annotations__", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + Py_DECREF(AnnAssign_annotations); + PyObject *For_annotations = PyDict_New(); + if (!For_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->For_type, "_field_types", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->For_type, "__annotations__", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + Py_DECREF(For_annotations); + PyObject *AsyncFor_annotations = PyDict_New(); + if (!AsyncFor_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "_field_types", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "__annotations__", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + Py_DECREF(AsyncFor_annotations); + PyObject *While_annotations = PyDict_New(); + if (!While_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(While_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->While_type, "_field_types", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->While_type, "__annotations__", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + Py_DECREF(While_annotations); + PyObject *If_annotations = PyDict_New(); + if (!If_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(If_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->If_type, "_field_types", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->If_type, "__annotations__", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + Py_DECREF(If_annotations); + PyObject *With_annotations = PyDict_New(); + if (!With_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "type_comment", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->With_type, "_field_types", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->With_type, "__annotations__", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + Py_DECREF(With_annotations); + PyObject *AsyncWith_annotations = PyDict_New(); + if (!AsyncWith_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "_field_types", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "__annotations__", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + Py_DECREF(AsyncWith_annotations); + PyObject *Match_annotations = PyDict_New(); + if (!Match_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Match_annotations, "subject", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + { + PyObject *type = state->match_case_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyDict_SetItemString(Match_annotations, "cases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Match_type, "_field_types", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Match_type, "__annotations__", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + Py_DECREF(Match_annotations); + PyObject *Raise_annotations = PyDict_New(); + if (!Raise_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "exc", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "cause", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Raise_type, "_field_types", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Raise_type, "__annotations__", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + Py_DECREF(Raise_annotations); + PyObject *Try_annotations = PyDict_New(); + if (!Try_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "finalbody", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Try_type, "_field_types", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Try_type, "__annotations__", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + Py_DECREF(Try_annotations); + PyObject *TryStar_annotations = PyDict_New(); + if (!TryStar_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "finalbody", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TryStar_type, "_field_types", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TryStar_type, "__annotations__", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + Py_DECREF(TryStar_annotations); + PyObject *Assert_annotations = PyDict_New(); + if (!Assert_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assert_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyDict_SetItemString(Assert_annotations, "msg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assert_type, "_field_types", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assert_type, "__annotations__", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + Py_DECREF(Assert_annotations); + PyObject *Import_annotations = PyDict_New(); + if (!Import_annotations) return 0; + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyDict_SetItemString(Import_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Import_type, "_field_types", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Import_type, "__annotations__", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + Py_DECREF(Import_annotations); + PyObject *ImportFrom_annotations = PyDict_New(); + if (!ImportFrom_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "module", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "level", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "_field_types", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "__annotations__", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + Py_DECREF(ImportFrom_annotations); + PyObject *Global_annotations = PyDict_New(); + if (!Global_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyDict_SetItemString(Global_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Global_type, "_field_types", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Global_type, "__annotations__", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + Py_DECREF(Global_annotations); + PyObject *Nonlocal_annotations = PyDict_New(); + if (!Nonlocal_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyDict_SetItemString(Nonlocal_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "_field_types", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "__annotations__", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + Py_DECREF(Nonlocal_annotations); + PyObject *Expr_annotations = PyDict_New(); + if (!Expr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expr_type, "_field_types", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expr_type, "__annotations__", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + Py_DECREF(Expr_annotations); + PyObject *Pass_annotations = PyDict_New(); + if (!Pass_annotations) return 0; + cond = PyObject_SetAttrString(state->Pass_type, "_field_types", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pass_type, "__annotations__", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + Py_DECREF(Pass_annotations); + PyObject *Break_annotations = PyDict_New(); + if (!Break_annotations) return 0; + cond = PyObject_SetAttrString(state->Break_type, "_field_types", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Break_type, "__annotations__", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + Py_DECREF(Break_annotations); + PyObject *Continue_annotations = PyDict_New(); + if (!Continue_annotations) return 0; + cond = PyObject_SetAttrString(state->Continue_type, "_field_types", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Continue_type, "__annotations__", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + Py_DECREF(Continue_annotations); + PyObject *BoolOp_annotations = PyDict_New(); + if (!BoolOp_annotations) return 0; + { + PyObject *type = state->boolop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BoolOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyDict_SetItemString(BoolOp_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BoolOp_type, "_field_types", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BoolOp_type, "__annotations__", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + Py_DECREF(BoolOp_annotations); + PyObject *NamedExpr_annotations = PyDict_New(); + if (!NamedExpr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "_field_types", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "__annotations__", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + Py_DECREF(NamedExpr_annotations); + PyObject *BinOp_annotations = PyDict_New(); + if (!BinOp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "right", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BinOp_type, "_field_types", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BinOp_type, "__annotations__", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + Py_DECREF(BinOp_annotations); + PyObject *UnaryOp_annotations = PyDict_New(); + if (!UnaryOp_annotations) return 0; + { + PyObject *type = state->unaryop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "operand", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "_field_types", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "__annotations__", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + Py_DECREF(UnaryOp_annotations); + PyObject *Lambda_annotations = PyDict_New(); + if (!Lambda_annotations) return 0; + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Lambda_type, "_field_types", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lambda_type, "__annotations__", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + Py_DECREF(Lambda_annotations); + PyObject *IfExp_annotations = PyDict_New(); + if (!IfExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->IfExp_type, "_field_types", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IfExp_type, "__annotations__", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + Py_DECREF(IfExp_annotations); + PyObject *Dict_annotations = PyDict_New(); + if (!Dict_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "keys", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Dict_type, "_field_types", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Dict_type, "__annotations__", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + Py_DECREF(Dict_annotations); + PyObject *Set_annotations = PyDict_New(); + if (!Set_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyDict_SetItemString(Set_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Set_type, "_field_types", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Set_type, "__annotations__", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + Py_DECREF(Set_annotations); + PyObject *ListComp_annotations = PyDict_New(); + if (!ListComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(ListComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyDict_SetItemString(ListComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ListComp_type, "_field_types", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ListComp_type, "__annotations__", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + Py_DECREF(ListComp_annotations); + PyObject *SetComp_annotations = PyDict_New(); + if (!SetComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(SetComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyDict_SetItemString(SetComp_annotations, "generators", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->SetComp_type, "_field_types", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->SetComp_type, "__annotations__", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + Py_DECREF(SetComp_annotations); + PyObject *DictComp_annotations = PyDict_New(); + if (!DictComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "key", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyDict_SetItemString(DictComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->DictComp_type, "_field_types", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->DictComp_type, "__annotations__", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + Py_DECREF(DictComp_annotations); + PyObject *GeneratorExp_annotations = PyDict_New(); + if (!GeneratorExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(GeneratorExp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyDict_SetItemString(GeneratorExp_annotations, "generators", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "_field_types", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "__annotations__", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + Py_DECREF(GeneratorExp_annotations); + PyObject *Await_annotations = PyDict_New(); + if (!Await_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Await_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Await_type, "_field_types", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Await_type, "__annotations__", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + Py_DECREF(Await_annotations); + PyObject *Yield_annotations = PyDict_New(); + if (!Yield_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyDict_SetItemString(Yield_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Yield_type, "_field_types", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Yield_type, "__annotations__", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + Py_DECREF(Yield_annotations); + PyObject *YieldFrom_annotations = PyDict_New(); + if (!YieldFrom_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(YieldFrom_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "_field_types", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "__annotations__", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + Py_DECREF(YieldFrom_annotations); + PyObject *Compare_annotations = PyDict_New(); + if (!Compare_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Compare_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->cmpop_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "ops", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "comparators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Compare_type, "_field_types", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Compare_type, "__annotations__", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + Py_DECREF(Compare_annotations); + PyObject *Call_annotations = PyDict_New(); + if (!Call_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Call_annotations, "func", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "keywords", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Call_type, "_field_types", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Call_type, "__annotations__", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + Py_DECREF(Call_annotations); + PyObject *FormattedValue_annotations = PyDict_New(); + if (!FormattedValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "conversion", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyDict_SetItemString(FormattedValue_annotations, "format_spec", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FormattedValue_type, "_field_types", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FormattedValue_type, + "__annotations__", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + Py_DECREF(FormattedValue_annotations); + PyObject *JoinedStr_annotations = PyDict_New(); + if (!JoinedStr_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyDict_SetItemString(JoinedStr_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "_field_types", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "__annotations__", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + Py_DECREF(JoinedStr_annotations); + PyObject *Constant_annotations = PyDict_New(); + if (!Constant_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Constant_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyDict_SetItemString(Constant_annotations, "kind", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Constant_type, "_field_types", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Constant_type, "__annotations__", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + Py_DECREF(Constant_annotations); + PyObject *Attribute_annotations = PyDict_New(); + if (!Attribute_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "attr", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Attribute_type, "_field_types", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Attribute_type, "__annotations__", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + Py_DECREF(Attribute_annotations); + PyObject *Subscript_annotations = PyDict_New(); + if (!Subscript_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "slice", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Subscript_type, "_field_types", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Subscript_type, "__annotations__", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + Py_DECREF(Subscript_annotations); + PyObject *Starred_annotations = PyDict_New(); + if (!Starred_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Starred_type, "_field_types", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Starred_type, "__annotations__", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + Py_DECREF(Starred_annotations); + PyObject *Name_annotations = PyDict_New(); + if (!Name_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "id", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Name_type, "_field_types", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Name_type, "__annotations__", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + Py_DECREF(Name_annotations); + PyObject *List_annotations = PyDict_New(); + if (!List_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyDict_SetItemString(List_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(List_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->List_type, "_field_types", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->List_type, "__annotations__", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + Py_DECREF(List_annotations); + PyObject *Tuple_annotations = PyDict_New(); + if (!Tuple_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyDict_SetItemString(Tuple_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Tuple_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Tuple_type, "_field_types", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Tuple_type, "__annotations__", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + Py_DECREF(Tuple_annotations); + PyObject *Slice_annotations = PyDict_New(); + if (!Slice_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "lower", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "upper", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "step", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Slice_type, "_field_types", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Slice_type, "__annotations__", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + Py_DECREF(Slice_annotations); + PyObject *Load_annotations = PyDict_New(); + if (!Load_annotations) return 0; + cond = PyObject_SetAttrString(state->Load_type, "_field_types", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Load_type, "__annotations__", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + Py_DECREF(Load_annotations); + PyObject *Store_annotations = PyDict_New(); + if (!Store_annotations) return 0; + cond = PyObject_SetAttrString(state->Store_type, "_field_types", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Store_type, "__annotations__", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + Py_DECREF(Store_annotations); + PyObject *Del_annotations = PyDict_New(); + if (!Del_annotations) return 0; + cond = PyObject_SetAttrString(state->Del_type, "_field_types", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Del_type, "__annotations__", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + Py_DECREF(Del_annotations); + PyObject *And_annotations = PyDict_New(); + if (!And_annotations) return 0; + cond = PyObject_SetAttrString(state->And_type, "_field_types", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->And_type, "__annotations__", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + Py_DECREF(And_annotations); + PyObject *Or_annotations = PyDict_New(); + if (!Or_annotations) return 0; + cond = PyObject_SetAttrString(state->Or_type, "_field_types", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Or_type, "__annotations__", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + Py_DECREF(Or_annotations); + PyObject *Add_annotations = PyDict_New(); + if (!Add_annotations) return 0; + cond = PyObject_SetAttrString(state->Add_type, "_field_types", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Add_type, "__annotations__", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + Py_DECREF(Add_annotations); + PyObject *Sub_annotations = PyDict_New(); + if (!Sub_annotations) return 0; + cond = PyObject_SetAttrString(state->Sub_type, "_field_types", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Sub_type, "__annotations__", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + Py_DECREF(Sub_annotations); + PyObject *Mult_annotations = PyDict_New(); + if (!Mult_annotations) return 0; + cond = PyObject_SetAttrString(state->Mult_type, "_field_types", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mult_type, "__annotations__", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + Py_DECREF(Mult_annotations); + PyObject *MatMult_annotations = PyDict_New(); + if (!MatMult_annotations) return 0; + cond = PyObject_SetAttrString(state->MatMult_type, "_field_types", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatMult_type, "__annotations__", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + Py_DECREF(MatMult_annotations); + PyObject *Div_annotations = PyDict_New(); + if (!Div_annotations) return 0; + cond = PyObject_SetAttrString(state->Div_type, "_field_types", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Div_type, "__annotations__", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + Py_DECREF(Div_annotations); + PyObject *Mod_annotations = PyDict_New(); + if (!Mod_annotations) return 0; + cond = PyObject_SetAttrString(state->Mod_type, "_field_types", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mod_type, "__annotations__", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + Py_DECREF(Mod_annotations); + PyObject *Pow_annotations = PyDict_New(); + if (!Pow_annotations) return 0; + cond = PyObject_SetAttrString(state->Pow_type, "_field_types", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pow_type, "__annotations__", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + Py_DECREF(Pow_annotations); + PyObject *LShift_annotations = PyDict_New(); + if (!LShift_annotations) return 0; + cond = PyObject_SetAttrString(state->LShift_type, "_field_types", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LShift_type, "__annotations__", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + Py_DECREF(LShift_annotations); + PyObject *RShift_annotations = PyDict_New(); + if (!RShift_annotations) return 0; + cond = PyObject_SetAttrString(state->RShift_type, "_field_types", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->RShift_type, "__annotations__", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + Py_DECREF(RShift_annotations); + PyObject *BitOr_annotations = PyDict_New(); + if (!BitOr_annotations) return 0; + cond = PyObject_SetAttrString(state->BitOr_type, "_field_types", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitOr_type, "__annotations__", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + Py_DECREF(BitOr_annotations); + PyObject *BitXor_annotations = PyDict_New(); + if (!BitXor_annotations) return 0; + cond = PyObject_SetAttrString(state->BitXor_type, "_field_types", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitXor_type, "__annotations__", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + Py_DECREF(BitXor_annotations); + PyObject *BitAnd_annotations = PyDict_New(); + if (!BitAnd_annotations) return 0; + cond = PyObject_SetAttrString(state->BitAnd_type, "_field_types", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitAnd_type, "__annotations__", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + Py_DECREF(BitAnd_annotations); + PyObject *FloorDiv_annotations = PyDict_New(); + if (!FloorDiv_annotations) return 0; + cond = PyObject_SetAttrString(state->FloorDiv_type, "_field_types", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FloorDiv_type, "__annotations__", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + Py_DECREF(FloorDiv_annotations); + PyObject *Invert_annotations = PyDict_New(); + if (!Invert_annotations) return 0; + cond = PyObject_SetAttrString(state->Invert_type, "_field_types", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Invert_type, "__annotations__", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + Py_DECREF(Invert_annotations); + PyObject *Not_annotations = PyDict_New(); + if (!Not_annotations) return 0; + cond = PyObject_SetAttrString(state->Not_type, "_field_types", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Not_type, "__annotations__", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + Py_DECREF(Not_annotations); + PyObject *UAdd_annotations = PyDict_New(); + if (!UAdd_annotations) return 0; + cond = PyObject_SetAttrString(state->UAdd_type, "_field_types", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UAdd_type, "__annotations__", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + Py_DECREF(UAdd_annotations); + PyObject *USub_annotations = PyDict_New(); + if (!USub_annotations) return 0; + cond = PyObject_SetAttrString(state->USub_type, "_field_types", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->USub_type, "__annotations__", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + Py_DECREF(USub_annotations); + PyObject *Eq_annotations = PyDict_New(); + if (!Eq_annotations) return 0; + cond = PyObject_SetAttrString(state->Eq_type, "_field_types", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Eq_type, "__annotations__", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + Py_DECREF(Eq_annotations); + PyObject *NotEq_annotations = PyDict_New(); + if (!NotEq_annotations) return 0; + cond = PyObject_SetAttrString(state->NotEq_type, "_field_types", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotEq_type, "__annotations__", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + Py_DECREF(NotEq_annotations); + PyObject *Lt_annotations = PyDict_New(); + if (!Lt_annotations) return 0; + cond = PyObject_SetAttrString(state->Lt_type, "_field_types", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lt_type, "__annotations__", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + Py_DECREF(Lt_annotations); + PyObject *LtE_annotations = PyDict_New(); + if (!LtE_annotations) return 0; + cond = PyObject_SetAttrString(state->LtE_type, "_field_types", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LtE_type, "__annotations__", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + Py_DECREF(LtE_annotations); + PyObject *Gt_annotations = PyDict_New(); + if (!Gt_annotations) return 0; + cond = PyObject_SetAttrString(state->Gt_type, "_field_types", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Gt_type, "__annotations__", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + Py_DECREF(Gt_annotations); + PyObject *GtE_annotations = PyDict_New(); + if (!GtE_annotations) return 0; + cond = PyObject_SetAttrString(state->GtE_type, "_field_types", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GtE_type, "__annotations__", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + Py_DECREF(GtE_annotations); + PyObject *Is_annotations = PyDict_New(); + if (!Is_annotations) return 0; + cond = PyObject_SetAttrString(state->Is_type, "_field_types", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Is_type, "__annotations__", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + Py_DECREF(Is_annotations); + PyObject *IsNot_annotations = PyDict_New(); + if (!IsNot_annotations) return 0; + cond = PyObject_SetAttrString(state->IsNot_type, "_field_types", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IsNot_type, "__annotations__", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + Py_DECREF(IsNot_annotations); + PyObject *In_annotations = PyDict_New(); + if (!In_annotations) return 0; + cond = PyObject_SetAttrString(state->In_type, "_field_types", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->In_type, "__annotations__", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + Py_DECREF(In_annotations); + PyObject *NotIn_annotations = PyDict_New(); + if (!NotIn_annotations) return 0; + cond = PyObject_SetAttrString(state->NotIn_type, "_field_types", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotIn_type, "__annotations__", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + Py_DECREF(NotIn_annotations); + PyObject *comprehension_annotations = PyDict_New(); + if (!comprehension_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "target", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "iter", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyDict_SetItemString(comprehension_annotations, "ifs", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "is_async", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->comprehension_type, "_field_types", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->comprehension_type, "__annotations__", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + Py_DECREF(comprehension_annotations); + PyObject *ExceptHandler_annotations = PyDict_New(); + if (!ExceptHandler_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "type", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "body", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "_field_types", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "__annotations__", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + Py_DECREF(ExceptHandler_annotations); + PyObject *arguments_annotations = PyDict_New(); + if (!arguments_annotations) return 0; + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "posonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "vararg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kw_defaults", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwarg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "defaults", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arguments_type, "_field_types", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arguments_type, "__annotations__", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + Py_DECREF(arguments_annotations); + PyObject *arg_annotations = PyDict_New(); + if (!arg_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(arg_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "annotation", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arg_type, "_field_types", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arg_type, "__annotations__", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + Py_DECREF(arg_annotations); + PyObject *keyword_annotations = PyDict_New(); + if (!keyword_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyDict_SetItemString(keyword_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(keyword_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->keyword_type, "_field_types", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->keyword_type, "__annotations__", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + Py_DECREF(keyword_annotations); + PyObject *alias_annotations = PyDict_New(); + if (!alias_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(alias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyDict_SetItemString(alias_annotations, "asname", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->alias_type, "_field_types", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->alias_type, "__annotations__", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + Py_DECREF(alias_annotations); + PyObject *withitem_annotations = PyDict_New(); + if (!withitem_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(withitem_annotations, "context_expr", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyDict_SetItemString(withitem_annotations, "optional_vars", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->withitem_type, "_field_types", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->withitem_type, "__annotations__", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + Py_DECREF(withitem_annotations); + PyObject *match_case_annotations = PyDict_New(); + if (!match_case_annotations) return 0; + { + PyObject *type = state->pattern_type; + Py_INCREF(type); + cond = PyDict_SetItemString(match_case_annotations, "pattern", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "guard", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->match_case_type, "_field_types", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->match_case_type, "__annotations__", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + Py_DECREF(match_case_annotations); + PyObject *MatchValue_annotations = PyDict_New(); + if (!MatchValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchValue_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchValue_type, "_field_types", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchValue_type, "__annotations__", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + Py_DECREF(MatchValue_annotations); + PyObject *MatchSingleton_annotations = PyDict_New(); + if (!MatchSingleton_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchSingleton_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, "_field_types", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, + "__annotations__", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + Py_DECREF(MatchSingleton_annotations); + PyObject *MatchSequence_annotations = PyDict_New(); + if (!MatchSequence_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchSequence_annotations, "patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "_field_types", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "__annotations__", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + Py_DECREF(MatchSequence_annotations); + PyObject *MatchMapping_annotations = PyDict_New(); + if (!MatchMapping_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "keys", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "rest", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "_field_types", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "__annotations__", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + Py_DECREF(MatchMapping_annotations); + PyObject *MatchClass_annotations = PyDict_New(); + if (!MatchClass_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchClass_annotations, "cls", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_attrs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchClass_type, "_field_types", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchClass_type, "__annotations__", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + Py_DECREF(MatchClass_annotations); + PyObject *MatchStar_annotations = PyDict_New(); + if (!MatchStar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchStar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchStar_type, "_field_types", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchStar_type, "__annotations__", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + Py_DECREF(MatchStar_annotations); + PyObject *MatchAs_annotations = PyDict_New(); + if (!MatchAs_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "pattern", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchAs_type, "_field_types", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchAs_type, "__annotations__", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + Py_DECREF(MatchAs_annotations); + PyObject *MatchOr_annotations = PyDict_New(); + if (!MatchOr_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchOr_annotations, "patterns", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchOr_type, "_field_types", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchOr_type, "__annotations__", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + Py_DECREF(MatchOr_annotations); + PyObject *TypeIgnore_annotations = PyDict_New(); + if (!TypeIgnore_annotations) return 0; + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "lineno", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "tag", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "_field_types", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "__annotations__", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + Py_DECREF(TypeIgnore_annotations); + PyObject *TypeVar_annotations = PyDict_New(); + if (!TypeVar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "bound", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVar_type, "_field_types", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVar_type, "__annotations__", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + Py_DECREF(TypeVar_annotations); + PyObject *ParamSpec_annotations = PyDict_New(); + if (!ParamSpec_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ParamSpec_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "_field_types", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "__annotations__", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + Py_DECREF(ParamSpec_annotations); + PyObject *TypeVarTuple_annotations = PyDict_New(); + if (!TypeVarTuple_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVarTuple_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "_field_types", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "__annotations__", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + Py_DECREF(TypeVarTuple_annotations); + + return 1; +} + + typedef struct { PyObject_HEAD @@ -860,7 +5026,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -869,6 +5035,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -888,6 +5061,11 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -900,13 +5078,14 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -914,15 +5093,91 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } } + else if ( + PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 + ) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types)) { + res = -1; + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, name); + } + goto set_remaining_cleanup; + } + if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + res = -1; + goto cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -934,14 +5189,75 @@ ast_type_reduce(PyObject *self, PyObject *unused) return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL, + *remaining_dict = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + // Serialize the fields as positional args if possible, because if we + // serialize them as a dict, during unpickling they are set only *after* + // the object is constructed, which will now trigger a DeprecationWarning + // if the AST type has required fields. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + remaining_dict = PyDict_Copy(dict); + Py_DECREF(dict); + if (!remaining_dict) { + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value = PyDict_GetItemWithError(remaining_dict, name); + if (!value) { + if (PyErr_Occurred()) { + goto cleanup; + } + break; + } + if (PyList_Append(positional_args, value) < 0) { + goto cleanup; + } + if (PyDict_DelItem(remaining_dict, name) < 0) { + goto cleanup; + } + Py_DECREF(name); + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple, + remaining_dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); } - return Py_BuildValue("O()", Py_TYPE(self)); +cleanup: + Py_XDECREF(fields); + Py_XDECREF(remaining_fields); + Py_XDECREF(remaining_dict); + Py_XDECREF(positional_args); + return result; } static PyMemberDef ast_type_members[] = { @@ -1939,6 +6255,9 @@ init_types(struct ast_state *state) "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return -1; + if (!add_ast_annotations(state)) { + return -1; + } return 0; } From 02beb9f0208d22fd8bd893e6e6ec813f7e51b235 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 27 Feb 2024 21:35:25 -0800 Subject: [PATCH 26/82] gh-116026: Try disabling rebuilds of dependents in Homebrew (#116027) --- .github/workflows/reusable-macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index ba62d9568c6b80..65b73cd791c75c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -17,6 +17,7 @@ jobs: HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 strategy: fail-fast: false From e72576c48b8be1e4f22c2f387f9769efa073c5be Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 28 Feb 2024 09:51:08 +0200 Subject: [PATCH 27/82] gh-115961: Improve tests for compressed file-like objects (GH-115963) * Increase coverage for compressed file-like objects initialized with a file name, an open file object, a file object opened by file descriptor, and a file-like object without name and mode attributes (io.BytesIO) * Increase coverage for name, fileno(), mode, readable(), writable(), seekable() in different modes and states * No longer skip tests with bytes names * Test objects implementing the path protocol, not just pathlib.Path. --- Lib/test/test_bz2.py | 138 ++++++++++++++++++++-- Lib/test/test_gzip.py | 183 +++++++++++++++++++++++++++-- Lib/test/test_lzma.py | 123 ++++++++++++++++--- Lib/test/test_tarfile.py | 21 +++- Lib/test/test_zipfile/test_core.py | 56 +++++++-- 5 files changed, 476 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc3698b4..772f0eacce28f5 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,19 +3,19 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink +from test.support.os_helper import unlink, FakePath import _compression import sys @@ -537,12 +537,136 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.fileno() + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.fileno() + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertRaises(io.UnsupportedOperation, f.fileno) + with self.assertRaises(ValueError): + f.fileno() + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.fileno(), raw.fileno()) + with self.assertRaises(ValueError): + f.fileno() + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) with BZ2File(bytes_filename, "rb") as f: @@ -552,7 +676,7 @@ def testOpenBytesFilename(self): self.assertEqual(f.read(), self.DATA) def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) with BZ2File(filename, "rb") as f: diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 128f933787a3f6..d220c7d06e50c9 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -5,7 +5,6 @@ import functools import io import os -import pathlib import struct import sys import unittest @@ -79,16 +78,18 @@ def test_write(self): f.close() def test_write_read_with_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.GzipFile(filename, 'w') as f: f.write(data1 * 50) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) with gzip.GzipFile(filename, 'a') as f: f.write(data1) with gzip.GzipFile(filename) as f: d = f.read() self.assertEqual(d, data1 * 51) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) # The following test_write_xy methods test that write accepts # the corresponding bytes-like object type as input @@ -472,13 +473,118 @@ def test_textio_readlines(self): with io.TextIOWrapper(f, encoding="ascii") as t: self.assertEqual(t.readlines(), lines) + def test_fileobj_with_name(self): + with open(self.filename, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + def test_fileobj_from_fdopen(self): # Issue #13781: Opening a GzipFile for writing fails when using a # fileobj created with os.fdopen(). - fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT) - with os.fdopen(fd, "wb") as f: - with gzip.GzipFile(fileobj=f, mode="w") as g: - pass + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + with os.fdopen(fd, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with os.fdopen(fd, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with os.fdopen(fd, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with os.fdopen(fd, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): gzip.GzipFile(self.filename, "wb").close() @@ -508,17 +614,69 @@ def test_fileobj_mode(self): def test_bytes_filename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with gzip.GzipFile(bytes_filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, bytes_filename) with gzip.GzipFile(bytes_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with gzip.GzipFile(str_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, str_filename) + + def test_fileobj_without_name(self): + bio = io.BytesIO() + with gzip.GzipFile(fileobj=bio, mode='wb') as f: + f.write(data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + bio.seek(0) + with gzip.GzipFile(fileobj=bio, mode='rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + + def test_fileobj_and_filename(self): + filename2 = self.filename + 'new' + with (open(self.filename, 'wb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f): + f.write(data1 * 50) + self.assertEqual(f.name, filename2) + with (open(self.filename, 'rb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f): + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, filename2) + # Sanity check that we are actually operating on the right file. + with gzip.GzipFile(self.filename, 'rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, self.filename) def test_decompress_limited(self): """Decompressed data buffering should be limited""" @@ -707,13 +865,16 @@ def test_binary_modes(self): self.assertEqual(file_data, uncompressed) def test_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.open(filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, self.filename) with gzip.open(filename, "ab") as f: f.write(data1) + self.assertEqual(f.name, self.filename) with gzip.open(filename) as f: self.assertEqual(f.read(), data1 * 51) + self.assertEqual(f.name, self.filename) def test_implicit_binary_modes(self): # Test implicit binary modes (no "b" or "t" in mode string). diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 65e6488c5d7b10..db290e139327e0 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -2,7 +2,6 @@ import array from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os -import pathlib import pickle import random import sys @@ -12,7 +11,7 @@ from test.support import _4G, bigmemtest from test.support.import_helper import import_module from test.support.os_helper import ( - TESTFN, unlink + TESTFN, unlink, FakePath ) lzma = import_module("lzma") @@ -548,7 +547,7 @@ def test_init(self): pass def test_init_with_PathLike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) @@ -585,11 +584,10 @@ def test_init_with_x_mode(self): self.addCleanup(unlink, TESTFN) for mode in ("x", "xb"): unlink(TESTFN) - with LZMAFile(TESTFN, mode): + with LZMAFile(TESTFN, mode) as f: pass with self.assertRaises(FileExistsError): - with LZMAFile(TESTFN, mode): - pass + LZMAFile(TESTFN, mode) def test_init_bad_mode(self): with self.assertRaises(ValueError): @@ -867,17 +865,59 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_from_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + def test_read_incomplete(self): with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: self.assertRaises(EOFError, f.read) @@ -1051,6 +1091,17 @@ def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1058,10 +1109,7 @@ def test_write_to_file(self): unlink(TESTFN) def test_write_to_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) @@ -1071,6 +1119,51 @@ def test_write_to_file_with_bytes_filename(self): finally: unlink(TESTFN) + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + def test_write_append_to_file(self): part1 = INPUT[:1024] part2 = INPUT[1024:1536] @@ -1276,7 +1369,7 @@ def test_filename(self): self.assertEqual(f.read(), INPUT * 2) def test_with_pathlike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 51f070e96047a6..a047780fdd7e17 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -507,14 +507,32 @@ def test_length_zero_header(self): with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')): pass - def test_extractfile_name(self): + def test_extractfile_attrs(self): # gh-74468: TarFile.name must name a file, not a parent archive. file = self.tar.getmember('ustar/regtype') with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + self.assertIs(fobj.closed, False) + self.assertIs(fobj.closed, True) + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) class MiscReadTestBase(CommonReadTest): + is_stream = False def requires_name_attribute(self): pass @@ -807,6 +825,7 @@ def requires_name_attribute(self): class StreamReadTest(CommonReadTest, unittest.TestCase): prefix="r|" + is_stream = True def test_read_through(self): # Issue #11224: A poorly designed _FileInFile.read() method diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 087fa8d65cc336..7d89f753448dd7 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4,7 +4,6 @@ import io import itertools import os -import pathlib import posixpath import struct import subprocess @@ -25,7 +24,7 @@ captured_stdout, captured_stderr, requires_subprocess ) from test.support.os_helper import ( - TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath ) @@ -160,7 +159,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -447,6 +446,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'r') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'r') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -578,17 +598,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -1285,6 +1304,21 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1487,7 +1521,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1545,7 +1579,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1580,7 +1614,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1893,7 +1927,7 @@ def test_is_zip_erroneous_file(self): fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -3013,7 +3047,7 @@ def test_from_file(self): self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) From d53560deb2c9ae12147201003fe63b266654ee21 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 28 Feb 2024 01:56:40 -0800 Subject: [PATCH 28/82] gh-105858: Expose some union-related objects as internal APIs (GH-116025) We now use these in the AST parsing code after gh-105880. A few comparable types (e.g., NoneType) are already exposed as internal APIs. --- Include/internal/pycore_unionobject.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index 87264635b6e1cf..6ece7134cdeca0 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -8,9 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyTypeObject _PyUnion_Type; +// For extensions created by test_peg_generator +PyAPI_DATA(PyTypeObject) _PyUnion_Type; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject *, PyObject *); + #define _PyUnion_Check(op) Py_IS_TYPE((op), &_PyUnion_Type) -extern PyObject *_Py_union_type_or(PyObject *, PyObject *); #define _PyGenericAlias_Check(op) PyObject_TypeCheck((op), &Py_GenericAliasType) extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); From 1752b51012269eaa35f7a28f162d18479a4f72aa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 28 Feb 2024 10:17:34 +0000 Subject: [PATCH 29/82] gh-115773: Add tests to exercise the _Py_DebugOffsets structure (#115774) --- Include/internal/pycore_runtime.h | 77 +-- Include/internal/pycore_runtime_init.h | 5 + Lib/test/test_external_inspection.py | 84 +++ Modules/Setup | 1 + Modules/Setup.stdlib.in | 1 + Modules/_testexternalinspection.c | 629 ++++++++++++++++++++ Tools/build/generate_stdlib_module_names.py | 1 + configure | 50 ++ configure.ac | 3 +- pyconfig.h.in | 3 + 10 files changed, 818 insertions(+), 36 deletions(-) create mode 100644 Lib/test/test_external_inspection.py create mode 100644 Modules/_testexternalinspection.c diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0c9c59e85b2fcf..dc6f6f100f7a92 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -55,74 +55,81 @@ typedef struct _Py_DebugOffsets { uint64_t version; // Runtime state offset; struct _runtime_state { - off_t finalizing; - off_t interpreters_head; + uint64_t finalizing; + uint64_t interpreters_head; } runtime_state; // Interpreter state offset; struct _interpreter_state { - off_t next; - off_t threads_head; - off_t gc; - off_t imports_modules; - off_t sysdict; - off_t builtins; - off_t ceval_gil; - off_t gil_runtime_state_locked; - off_t gil_runtime_state_holder; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; } interpreter_state; // Thread state offset; struct _thread_state{ - off_t prev; - off_t next; - off_t interp; - off_t current_frame; - off_t thread_id; - off_t native_thread_id; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; } thread_state; // InterpreterFrame offset; struct _interpreter_frame { - off_t previous; - off_t executable; - off_t instr_ptr; - off_t localsplus; - off_t owner; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; } interpreter_frame; // CFrame offset; struct _cframe { - off_t current_frame; - off_t previous; + uint64_t current_frame; + uint64_t previous; } cframe; // Code object offset; struct _code_object { - off_t filename; - off_t name; - off_t linetable; - off_t firstlineno; - off_t argcount; - off_t localsplusnames; - off_t localspluskinds; - off_t co_code_adaptive; + uint64_t filename; + uint64_t name; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; } code_object; // PyObject offset; struct _pyobject { - off_t ob_type; + uint64_t ob_type; } pyobject; // PyTypeObject object offset; struct _type_object { - off_t tp_name; + uint64_t tp_name; } type_object; // PyTuple object offset; struct _tuple_object { - off_t ob_item; + uint64_t ob_item; } tuple_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t state; + uint64_t length; + size_t asciiobject_size; + } unicode_object; } _Py_DebugOffsets; /* Full Python runtime state */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index d093047d4bc09d..cc47b9a82e2879 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -83,6 +83,11 @@ extern PyTypeObject _PyExc_MemoryError; .tuple_object = { \ .ob_item = offsetof(PyTupleObject, ob_item), \ }, \ + .unicode_object = { \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py new file mode 100644 index 00000000000000..86c07de507e39c --- /dev/null +++ b/Lib/test/test_external_inspection.py @@ -0,0 +1,84 @@ +import unittest +import os +import textwrap +import importlib +import sys +from test.support import os_helper, SHORT_TIMEOUT +from test.support.script_helper import make_script + +import subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _testexternalinspection import get_stack_trace +except ImportError: + unittest.skip("Test only runs when _testexternalinspection is available") + +def _make_test_script(script_dir, script_basename, source): + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + +class TestGetStackTrace(unittest.TestCase): + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import time, sys, os + def bar(): + for x in range(100): + if x == 50: + baz() + def baz(): + foo() + + def foo(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(1000) + + bar() + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + 'foo', + 'baz', + 'bar', + '' + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_self_trace(self): + stack_trace = get_stack_trace(os.getpid()) + self.assertEqual(stack_trace[0], "test_self_trace") + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/Setup b/Modules/Setup index 8ad9a5aebbfcaa..cd1cf24c25d406 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -285,6 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c +#_testexternalinspection _testexternalinspection.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index e98775a4808765..73b082691a3fd4 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -171,6 +171,7 @@ @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c new file mode 100644 index 00000000000000..19ee3b8dd1428d --- /dev/null +++ b/Modules/_testexternalinspection.c @@ -0,0 +1,629 @@ +#define _GNU_SOURCE + +#ifdef __linux__ +# include +# include +# if INTPTR_MAX == INT64_MAX +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Shdr Elf64_Shdr +# define Elf_Phdr Elf64_Phdr +# else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Shdr Elf32_Shdr +# define Elf_Phdr Elf32_Phdr +# endif +# include +#endif + +#ifdef __APPLE__ +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif +#include "Python.h" +#include + +#ifndef HAVE_PROCESS_VM_READV +# define HAVE_PROCESS_VM_READV 0 +#endif + +#ifdef __APPLE__ +static void* +analyze_macho64(mach_port_t proc_ref, void* base, void* map) +{ + struct mach_header_64* hdr = (struct mach_header_64*)map; + int ncmds = hdr->ncmds; + + int cmd_cnt = 0; + struct segment_command_64* cmd = map + sizeof(struct mach_header_64); + + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + mach_vm_address_t address = (mach_vm_address_t)base; + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { + while (cmd->filesize != size) { + address += size; + if (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] + &count, + &object_name) + != KERN_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + return NULL; + } + } + base = (void*)address - cmd->vmaddr; + + int nsects = cmd->nsects; + struct section_64* sec = + (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + for (int j = 0; j < nsects; j++) { + if (strcmp(sec[j].sectname, "PyRuntime") == 0) { + return base + sec[j].addr; + } + } + cmd_cnt++; + } + + cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); + } + return NULL; +} + +static void* +analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +{ + int fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + return NULL; + } + + struct stat fs; + if (fstat(fd, &fs) == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + close(fd); + return NULL; + } + + void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + close(fd); + return NULL; + } + + void* result = NULL; + + struct mach_header_64* hdr = (struct mach_header_64*)map; + switch (hdr->magic) { + case MH_MAGIC: + case MH_CIGAM: + case FAT_MAGIC: + case FAT_CIGAM: + PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + break; + case MH_MAGIC_64: + case MH_CIGAM_64: + result = analyze_macho64(proc_ref, base, map); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + break; + } + + munmap(map, fs.st_size); + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + return result; +} + +static mach_port_t +pid_to_task(pid_t pid) +{ + mach_port_t task; + kern_return_t result; + + result = task_for_pid(mach_task_self(), pid, &task); + if (result != KERN_SUCCESS) { + PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + return 0; + } + return task; +} + +static void* +get_py_runtime_macos(pid_t pid) +{ + mach_vm_address_t address = 0; + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + mach_port_t proc_ref = pid_to_task(pid); + if (proc_ref == 0) { + PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + return NULL; + } + + int match_found = 0; + char map_filename[MAXPATHLEN + 1]; + void* result_address = NULL; + while (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, + &count, + &object_name) + == KERN_SUCCESS) + { + int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + if (path_len == 0) { + address += size; + continue; + } + + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + break; + } + + address += size; + } + return result_address; +} +#endif + +#ifdef __linux__ +void* +find_python_map_start_address(pid_t pid, char* result_filename) +{ + char maps_file_path[64]; + sprintf(maps_file_path, "/proc/%d/maps", pid); + + FILE* maps_file = fopen(maps_file_path, "r"); + if (maps_file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + int match_found = 0; + + char line[256]; + char map_filename[PATH_MAX]; + void* result_address = 0; + while (fgets(line, sizeof(line), maps_file) != NULL) { + unsigned long start_address = 0; + sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + break; + } + } + + fclose(maps_file); + + if (!match_found) { + map_filename[0] = '\0'; + } + + return result_address; +} + +void* +get_py_runtime_linux(pid_t pid) +{ + char elf_file[256]; + void* start_address = (void*)find_python_map_start_address(pid, elf_file); + + if (start_address == 0) { + PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); + return NULL; + } + + void* result = NULL; + void* file_memory = NULL; + + int fd = open(elf_file, O_RDONLY); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + struct stat file_stats; + if (fstat(fd, &file_stats) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (file_memory == MAP_FAILED) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + + Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; + char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); + + Elf_Shdr* py_runtime_section = NULL; + for (int i = 0; i < elf_header->e_shnum; i++) { + if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { + py_runtime_section = §ion_header_table[i]; + break; + } + } + + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment + Elf_Phdr* first_load_segment = NULL; + for (int i = 0; i < elf_header->e_phnum; i++) { + if (program_header_table[i].p_type == PT_LOAD) { + first_load_segment = &program_header_table[i]; + break; + } + } + + if (py_runtime_section != NULL && first_load_segment != NULL) { + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + py_runtime_section->sh_addr - elf_load_addr; + } + +exit: + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + if (file_memory != NULL) { + munmap(file_memory, file_stats.st_size); + } + return result; +} +#endif + +ssize_t +read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +{ + ssize_t total_bytes_read = 0; +#ifdef __linux__ + struct iovec local[1]; + struct iovec remote[1]; + ssize_t result = 0; + ssize_t read = 0; + + do { + local[0].iov_base = dst + result; + local[0].iov_len = len - result; + remote[0].iov_base = (void*)(remote_address + result); + remote[0].iov_len = len - result; + + read = process_vm_readv(pid, local, 1, remote, 1, 0); + if (read < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += read; + } while ((size_t)read != local[0].iov_len); + total_bytes_read = result; +#elif defined(__APPLE__) + ssize_t result = -1; + kern_return_t kr = mach_vm_read_overwrite( + pid_to_task(pid), + (mach_vm_address_t)remote_address, + len, + (mach_vm_address_t)dst, + (mach_vm_size_t*)&result); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + } + return -1; + } + total_bytes_read = len; +#else + return -1; +#endif + return total_bytes_read; +} + +int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +{ + Py_ssize_t len; + ssize_t bytes_read = + read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + if (bytes_read == -1) { + return -1; + } + if (len >= size) { + PyErr_SetString(PyExc_RuntimeError, "Buffer too small"); + return -1; + } + size_t offset = debug_offsets->unicode_object.asciiobject_size; + bytes_read = read_memory(pid, address + offset, len, buffer); + if (bytes_read == -1) { + return -1; + } + buffer[len] = '\0'; + return 0; +} + +void* +get_py_runtime(pid_t pid) +{ +#if defined(__linux__) + return get_py_runtime_linux(pid); +#elif defined(__APPLE__) + return get_py_runtime_macos(pid); +#else + return NULL; +#endif +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + void* address_of_function_name; + read_memory( + pid, + (void*)(address + offsets->code_object.name), + sizeof(void*), + &address_of_function_name); + + if (address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + char function_name[256]; + if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + return -1; + } + + PyObject* py_function_name = PyUnicode_FromString(function_name); + if (py_function_name == NULL) { + return -1; + } + + if (PyList_Append(result, py_function_name) == -1) { + Py_DECREF(py_function_name); + return -1; + } + Py_DECREF(py_function_name); + + return 0; +} + +static int +parse_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + ssize_t bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.previous), + sizeof(void*), + previous_frame); + if (bytes_read == -1) { + return -1; + } + + char owner; + bytes_read = + read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + if (bytes_read < 0) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + void* address_of_code_object; + bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.executable), + sizeof(void*), + &address_of_code_object); + if (bytes_read == -1) { + return -1; + } + + if (address_of_code_object == NULL) { + return 0; + } + return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame); +} + +static PyObject* +get_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { + return NULL; + } + + void* runtime_start_address = get_py_runtime(pid); + if (runtime_start_address == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return NULL; + } + size_t size = sizeof(struct _Py_DebugOffsets); + struct _Py_DebugOffsets local_debug_offsets; + + ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); + if (bytes_read == -1) { + return NULL; + } + off_t thread_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + + void* address_of_interpreter_state; + bytes_read = read_memory( + pid, + (void*)(runtime_start_address + thread_state_list_head), + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return NULL; + } + + if (address_of_interpreter_state == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return NULL; + } + + void* address_of_thread; + bytes_read = read_memory( + pid, + (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return NULL; + } + + PyObject* result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // No Python frames are available for us (can happen at tear-down). + if (address_of_thread != NULL) { + void* address_of_current_frame; + (void)read_memory( + pid, + (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), + sizeof(void*), + &address_of_current_frame); + while (address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + } + + return result; +} + +static PyMethodDef methods[] = { + {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC +PyInit__testexternalinspection(void) +{ + PyObject* mod = PyModule_Create(&module); + int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 5dce4e042d1eb4..588dfda50658be 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,6 +34,7 @@ '_testinternalcapi', '_testmultiphase', '_testsinglephase', + '_testexternalinspection', '_xxsubinterpreters', '_xxinterpchannels', '_xxinterpqueues', diff --git a/configure b/configure index b565cdbfae1327..c204c9eb499559 100755 --- a/configure +++ b/configure @@ -661,6 +661,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE +MODULE__TESTEXTERNALINSPECTION_FALSE +MODULE__TESTEXTERNALINSPECTION_TRUE MODULE__TESTMULTIPHASE_FALSE MODULE__TESTMULTIPHASE_TRUE MODULE__TESTIMPORTMULTIPLE_FALSE @@ -17997,6 +17999,12 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "process_vm_readv" "ac_cv_func_process_vm_readv" +if test "x$ac_cv_func_process_vm_readv" = xyes +then : + printf "%s\n" "#define HAVE_PROCESS_VM_READV 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes @@ -30688,6 +30696,44 @@ fi printf "%s\n" "$py_cv_module__testmultiphase" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 +printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } + if test "$py_cv_module__testexternalinspection" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testexternalinspection=yes +else $as_nop + py_cv_module__testexternalinspection=missing +fi +else $as_nop + py_cv_module__testexternalinspection=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" + if test "x$py_cv_module__testexternalinspection" = xyes +then : + + + + +fi + if test "$py_cv_module__testexternalinspection" = yes; then + MODULE__TESTEXTERNALINSPECTION_TRUE= + MODULE__TESTEXTERNALINSPECTION_FALSE='#' +else + MODULE__TESTEXTERNALINSPECTION_TRUE='#' + MODULE__TESTEXTERNALINSPECTION_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 +printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 printf %s "checking for stdlib extension module xxsubtype... " >&6; } if test "$py_cv_module_xxsubtype" != "n/a" @@ -31300,6 +31346,10 @@ if test -z "${MODULE__TESTMULTIPHASE_TRUE}" && test -z "${MODULE__TESTMULTIPHASE as_fn_error $? "conditional \"MODULE__TESTMULTIPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then as_fn_error $? "conditional \"MODULE_XXSUBTYPE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 0694ef06364f82..702e588357789c 100644 --- a/configure.ac +++ b/configure.ac @@ -4920,7 +4920,7 @@ AC_CHECK_FUNCS([ \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ @@ -7581,6 +7581,7 @@ PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], diff --git a/pyconfig.h.in b/pyconfig.h.in index 63a3437ebb3eac..b93cac9d23c796 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -933,6 +933,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H +/* Define to 1 if you have the `process_vm_readv' function. */ +#undef HAVE_PROCESS_VM_READV + /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES From 3b63d0769f49171f53e9cecc686fa01a383bd4b1 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 28 Feb 2024 13:04:23 +0200 Subject: [PATCH 30/82] gh-116030: test_unparse: Add ``ctx`` argument to ``ast.Name`` calls (#116031) --- Lib/test/test_unparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 106704ba8c9c2d..bb15f64c59dbd1 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -370,13 +370,13 @@ def test_slices(self): self.check_ast_roundtrip("a[i:j, k]") def test_invalid_raise(self): - self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X"))) + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) def test_invalid_fstring_value(self): self.check_invalid( ast.JoinedStr( values=[ - ast.Name(id="test"), + ast.Name(id="test", ctx=ast.Load()), ast.Constant(value="test") ] ) @@ -718,7 +718,7 @@ def test_function_with_type_params_and_bound(self): body=[ast.Pass()], decorator_list=[], returns=None, - type_params=[ast.TypeVar("T", bound=ast.Name("int"))], + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], ) ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") From 7acf1fb5a70776429bd99e741d69471eb2d1c1bb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Feb 2024 12:53:48 +0100 Subject: [PATCH 31/82] gh-114911: Add CPUStopwatch test helper (GH-114912) A few of our tests measure the time of CPU-bound operation, mainly to avoid quadratic or worse behaviour. Add a helper to ignore GC and time spent in other processes. --- Lib/test/support/__init__.py | 40 +++++++++++++++++++++++++ Lib/test/test_int.py | 58 ++++++++++++++++-------------------- Lib/test/test_re.py | 19 ++++++------ 3 files changed, 75 insertions(+), 42 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 1d03ec0f5bd12b..401b2ce1fe213c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2381,6 +2381,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of + other processes). + + N.B.: + - This *includes* time spent in other threads. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.rseolution if. + + Usage: + + with ProcessStopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.process_time + clock_info = time.get_clock_info('process_time') + if get_time() <= 0: # some platforms like WASM lack process_time() + get_time = time.monotonic + clock_info = time.get_clock_info('monotonic') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 0bf55facad9fed..47fc50a0e20349 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -664,84 +664,78 @@ def test_denial_of_service_prevented_int_to_str(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 50_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. digits = 78_268 - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): huge_decimal = str(huge_int) - seconds_to_convert = get_time() - start self.assertEqual(len(huge_decimal), digits) # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') # We test with the limit almost at the size needed to check performance. # The performant limit check is slightly fuzzy, give it a some room. with support.adjust_int_max_str_digits(int(.995 * digits)): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): str(huge_int) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds said Zen based cloud VM. str(extra_huge_int) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_denial_of_service_prevented_str_to_int(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 100_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic digits = 133700 huge = '8'*digits - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): int(huge) - seconds_to_convert = get_time() - start # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') with support.adjust_int_max_str_digits(digits - 1): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): int(huge) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge = '7'*1_200_000 - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds in the Zen based cloud VM. int(extra_huge) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_power_of_two_bases_unlimited(self): """The limit does not apply to power of 2 bases.""" diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 993a7d6e264a1f..b1ac22c28cf7c1 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,7 +1,7 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch) import locale import re import string @@ -2284,17 +2284,16 @@ def test_bug_40736(self): def test_search_anchor_at_beginning(self): s = 'x'*10**7 - start = time.perf_counter() - for p in r'\Ay', r'^y': - self.assertIsNone(re.search(p, s)) - self.assertEqual(re.split(p, s), [s]) - self.assertEqual(re.findall(p, s), []) - self.assertEqual(list(re.finditer(p, s)), []) - self.assertEqual(re.sub(p, '', s), s) - t = time.perf_counter() - start + with CPUStopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) # Without optimization it takes 1 second on my computer. # With optimization -- 0.0003 seconds. - self.assertLess(t, 0.1) + self.assertLess(stopwatch.seconds, 0.1) def test_possessive_quantifiers(self): """Test Possessive Quantifiers From a71e32ce8e183023fc1ee401c22ebe35e4832f09 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Feb 2024 14:03:50 +0100 Subject: [PATCH 32/82] gh-78612: Mark up eval() using param list (#115212) Also mention that the 'expression' parameter can be a string. --- Doc/library/functions.rst | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a4852b922b65b3..e598ef423de497 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -526,9 +526,20 @@ are always available. They are listed here in alphabetical order. .. function:: eval(expression, globals=None, locals=None) - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + :param expression: + A Python expression. + :type expression: :class:`str` | :ref:`code object ` + + :param globals: + The global namespace (default: ``None``). + :type globals: :class:`dict` | ``None`` + + :param locals: + The local namespace (default: ``None``). + :type locals: :term:`mapping` | ``None`` + + :returns: The result of the evaluated expression. + :raises: Syntax errors are reported as exceptions. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* @@ -545,8 +556,7 @@ are always available. They are listed here in alphabetical order. :term:`nested scopes ` (non-locals) in the enclosing environment. - The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: + Example: >>> x = 1 >>> eval('x+1') From 449c6da2bdc5c6aa5e096aa550a4ba377b85db46 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 28 Feb 2024 14:35:41 +0100 Subject: [PATCH 33/82] gh-115765: Don't use deprecated AC_EGREP_* macros in configure.ac (#116016) Rewrite using AX_CHECK_DEFINE and AC_CHECK_TYPES. --- aclocal.m4 | 74 +++++++++++ configure | 360 +++++++++++++++++++++++++++++++++++--------------- configure.ac | 95 +++++-------- pyconfig.h.in | 5 +- 4 files changed, 363 insertions(+), 171 deletions(-) diff --git a/aclocal.m4 b/aclocal.m4 index 09ae5d1aa8a608..832aec19f48f17 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -150,6 +150,80 @@ AS_VAR_IF(CACHEVAR,yes, AS_VAR_POPDEF([CACHEVAR])dnl ])dnl AX_CHECK_COMPILE_FLAGS +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_define.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CHECK_DEFINE([symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# AX_CHECK_DEFINE([includes],[symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# +# DESCRIPTION +# +# Complements AC_CHECK_FUNC but it does not check for a function but for a +# define to exist. Consider a usage like: +# +# AC_CHECK_DEFINE(__STRICT_ANSI__, CFLAGS="$CFLAGS -D_XOPEN_SOURCE=500") +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +AU_ALIAS([AC_CHECK_DEFINED], [AC_CHECK_DEFINE]) +AC_DEFUN([AC_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$1])dnl +AC_CACHE_CHECK([for $1 defined], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ + #ifdef $1 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$2], [$3])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AU_ALIAS([AX_CHECK_DEFINED], [AX_CHECK_DEFINE]) +AC_DEFUN([AX_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$2_$1])dnl +AC_CACHE_CHECK([for $2 defined in $1], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <$1>]], [[ + #ifdef $2 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AC_DEFUN([AX_CHECK_FUNC], +[AS_VAR_PUSHDEF([ac_var], [ac_cv_func_$2])dnl +AC_CACHE_CHECK([for $2], ac_var, +dnl AC_LANG_FUNC_LINK_TRY +[AC_LINK_IFELSE([AC_LANG_PROGRAM([$1 + #undef $2 + char $2 ();],[ + char (*f) () = $2; + return f != $2; ])], + [AS_VAR_SET(ac_var, yes)], + [AS_VAR_SET(ac_var, no)])]) +AS_IF([test AS_VAR_GET(ac_var) = yes], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +])# AC_CHECK_FUNC + # =========================================================================== # https://www.gnu.org/software/autoconf-archive/ax_check_openssl.html # =========================================================================== diff --git a/configure b/configure index c204c9eb499559..f431c5dd15ec4a 100755 --- a/configure +++ b/configure @@ -11297,42 +11297,22 @@ then : fi -# checks for typedefs - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_t in time.h" >&5 -printf %s "checking for clock_t in time.h... " >&6; } -if test ${ac_cv_clock_t_time_h+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "clock_t" >/dev/null 2>&1 +# Check for clock_t in time.h. +ac_fn_c_check_type "$LINENO" "clock_t" "ac_cv_type_clock_t" "#include +" +if test "x$ac_cv_type_clock_t" = xyes then : - ac_cv_clock_t_time_h=yes -else $as_nop - ac_cv_clock_t_time_h=no -fi -rm -rf conftest* +printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_clock_t_time_h" >&5 -printf "%s\n" "$ac_cv_clock_t_time_h" >&6; } -if test "x$ac_cv_clock_t_time_h" = xno -then : +else $as_nop printf "%s\n" "#define clock_t long" >>confdefs.h - fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for makedev" >&5 printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} @@ -11534,6 +11514,7 @@ printf "%s\n" "#define size_t unsigned int" >>confdefs.h fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uid_t in sys/types.h" >&5 printf %s "checking for uid_t in sys/types.h... " >&6; } if test ${ac_cv_type_uid_t+y} @@ -16184,24 +16165,47 @@ else # (e.g. gnu pth with pthread emulation) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS in unistd.h" >&5 printf %s "checking for _POSIX_THREADS in unistd.h... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS defined in unistd.h" >&5 +printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } +if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _POSIX_THREADS -yes -#endif +int +main (void) +{ + #ifdef _POSIX_THREADS + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined__POSIX_THREADS_unistd_h=yes +else $as_nop + ac_cv_defined__POSIX_THREADS_unistd_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 +printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } +if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes else $as_nop unistd_defines_pthreads=no fi -rm -rf conftest* - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16708,59 +16712,131 @@ printf %s "checking ipv6 stack type... " >&6; } do case $i in inria) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for IPV6_INRIA_VERSION defined in netinet/in.h" >&5 +printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef IPV6_INRIA_VERSION -yes -#endif +int +main (void) +{ + + #ifdef IPV6_INRIA_VERSION + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes +else $as_nop + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } +if test $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h != "no" then : ipv6type=$i fi -rm -rf conftest* - ;; kame) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __KAME__ defined in netinet/in.h" >&5 +printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined___KAME___netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __KAME__ -yes -#endif +int +main (void) +{ + + #ifdef __KAME__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes + ac_cv_defined___KAME___netinet_in_h=yes +else $as_nop + ac_cv_defined___KAME___netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } +if test $ac_cv_defined___KAME___netinet_in_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-glibc) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __GLIBC__ defined in features.h" >&5 +printf %s "checking for __GLIBC__ defined in features.h... " >&6; } +if test ${ac_cv_defined___GLIBC___features_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -#endif +int +main (void) +{ + + #ifdef __GLIBC__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6trylibc=yes + ac_cv_defined___GLIBC___features_h=yes +else $as_nop + ac_cv_defined___GLIBC___features_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 +printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } +if test $ac_cv_defined___GLIBC___features_h != "no" +then : + ipv6type=$i + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-inet6) if test -d /usr/inet6; then @@ -16779,62 +16855,134 @@ rm -rf conftest* fi ;; toshiba) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _TOSHIBA_INET6 defined in sys/param.h" >&5 +printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _TOSHIBA_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _TOSHIBA_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__TOSHIBA_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; v6d) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h" >&5 +printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h... " >&6; } +if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __V6D__ -yes -#endif +int +main (void) +{ + + #ifdef __V6D__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes +else $as_nop + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 +printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } +if test $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h != "no" +then : + ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" fi -rm -rf conftest* - ;; zeta) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _ZETA_MINAMI_INET6 defined in sys/param.h" >&5 +printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _ZETA_MINAMI_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _ZETA_MINAMI_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; esac if test "$ipv6type" != "unknown"; then diff --git a/configure.ac b/configure.ac index 702e588357789c..a5abb06a829413 100644 --- a/configure.ac +++ b/configure.ac @@ -2895,15 +2895,11 @@ AC_CHECK_HEADERS( #endif ]) -# checks for typedefs -AC_CACHE_CHECK([for clock_t in time.h], [ac_cv_clock_t_time_h], [ - AC_EGREP_HEADER([clock_t], [time.h], [ac_cv_clock_t_time_h=yes], [ac_cv_clock_t_time_h=no]) -]) -dnl checks for "no" -AS_VAR_IF([ac_cv_clock_t_time_h], [no], [ - AC_DEFINE([clock_t], [long], - [Define to 'long' if doesn't define.]) -]) +# Check for clock_t in time.h. +AC_CHECK_TYPES([clock_t], [], + [AC_DEFINE([clock_t], [long], + [Define to 'long' if does not define clock_t.])], + [@%:@include ]) AC_CACHE_CHECK([for makedev], [ac_cv_func_makedev], [ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -4331,13 +4327,9 @@ else # define _POSIX_THREADS in unistd.h. Some apparently don't # (e.g. gnu pth with pthread emulation) AC_MSG_CHECKING([for _POSIX_THREADS in unistd.h]) - AC_EGREP_CPP([yes], - [ -#include -#ifdef _POSIX_THREADS -yes -#endif - ], unistd_defines_pthreads=yes, unistd_defines_pthreads=no) + AX_CHECK_DEFINE([unistd.h], [_POSIX_THREADS], + [unistd_defines_pthreads=yes], + [unistd_defines_pthreads=no]) AC_MSG_RESULT([$unistd_defines_pthreads]) AC_DEFINE([_REENTRANT]) @@ -4517,34 +4509,21 @@ if test "$ipv6" = yes -a "$cross_compiling" = no; then case $i in inria) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef IPV6_INRIA_VERSION -yes -@%:@endif], - [ipv6type=$i]) + AX_CHECK_DEFINE([netinet/in.h], [IPV6_INRIA_VERSION], [ipv6type=$i]) ;; kame) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef __KAME__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes]) + AX_CHECK_DEFINE([netinet/in.h], [__KAME__], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes]) ;; linux-glibc) - dnl http://www.v6.linux.or.jp/ - AC_EGREP_CPP([yes], [ -#include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -@%:@endif], - [ipv6type=$i; - ipv6trylibc=yes]) + dnl Advanced IPv6 support was added to glibc 2.1 in 1999. + AX_CHECK_DEFINE([features.h], [__GLIBC__], + [ipv6type=$i + ipv6trylibc=yes]) ;; linux-inet6) dnl http://www.v6.linux.or.jp/ @@ -4564,35 +4543,23 @@ yes fi ;; toshiba) - AC_EGREP_CPP([yes], [ -#include -#ifdef _TOSHIBA_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_TOSHIBA_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; v6d) - AC_EGREP_CPP([yes], [ -#include -#ifdef __V6D__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) + AX_CHECK_DEFINE([/usr/local/v6/include/sys/v6config.h], [__V6D__], + [ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) ;; zeta) - AC_EGREP_CPP([yes], [ -#include -#ifdef _ZETA_MINAMI_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_ZETA_MINAMI_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; esac if test "$ipv6type" != "unknown"; then diff --git a/pyconfig.h.in b/pyconfig.h.in index b93cac9d23c796..e28baef51d5737 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -157,6 +157,9 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if the system has the type `clock_t'. */ +#undef HAVE_CLOCK_T + /* Define to 1 if you have the `closefrom' function. */ #undef HAVE_CLOSEFROM @@ -1941,7 +1944,7 @@ /* Define on FreeBSD to activate all library features */ #undef __BSD_VISIBLE -/* Define to 'long' if doesn't define. */ +/* Define to 'long' if does not define clock_t. */ #undef clock_t /* Define to empty if `const' does not conform to ANSI C. */ From 647053fed182066d3b8c934fb0bf52ee48ff3911 Mon Sep 17 00:00:00 2001 From: Jan Max Meyer Date: Wed, 28 Feb 2024 14:54:12 +0100 Subject: [PATCH 34/82] doc: Use super() in subclassed JSONEncoder examples (GH-115565) Replace calls to `json.JSONEncoder.default(self, obj)` by `super().default(obj)` within the examples of the documentation. --- Doc/library/json.rst | 4 ++-- Lib/json/encoder.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 0ce4b697145cb3..c82ff9dc325b4c 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -106,7 +106,7 @@ Extending :class:`JSONEncoder`:: ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) + ... return super().default(obj) ... >>> json.dumps(2 + 1j, cls=ComplexEncoder) '[2.0, 1.0]' @@ -504,7 +504,7 @@ Encoders and Decoders else: return list(iterable) # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) + return super().default(o) .. method:: encode(o) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 45f547741885a8..597849eca0524a 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -174,7 +174,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' From 9578288a3e5a7f42d1f3bec139c0c85b87775c90 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 28 Feb 2024 13:58:25 +0000 Subject: [PATCH 35/82] gh-116012: Preserve GetLastError() across calls to TlsGetValue on Windows (GH-116014) --- .../2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst | 1 + Python/pystate.c | 9 --------- Python/thread_nt.h | 7 ++++++- 3 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst diff --git a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst new file mode 100644 index 00000000000000..a55e5b1c7b566d --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst @@ -0,0 +1 @@ +Ensure the value of ``GetLastError()`` is preserved across GIL operations. diff --git a/Python/pystate.c b/Python/pystate.c index a80c1b7fb9c866..a370fff857af85 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2528,16 +2528,7 @@ PyGILState_Check(void) return 0; } -#ifdef MS_WINDOWS - int err = GetLastError(); -#endif - PyThreadState *tcur = gilstate_tss_get(runtime); - -#ifdef MS_WINDOWS - SetLastError(err); -#endif - return (tstate == tcur); } diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 7922b2d7e84845..9dca833ff203ca 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -513,5 +513,10 @@ void * PyThread_tss_get(Py_tss_t *key) { assert(key != NULL); - return TlsGetValue(key->_key); + int err = GetLastError(); + void *r = TlsGetValue(key->_key); + if (r || !GetLastError()) { + SetLastError(err); + } + return r; } From 0a61e237009bf6b833e13ac635299ee063377699 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 28 Feb 2024 07:21:42 -0800 Subject: [PATCH 36/82] gh-107674: Improve performance of `sys.settrace` (GH-114986) --- ...-02-04-07-45-29.gh-issue-107674.q8mCmi.rst | 1 + Python/bytecodes.c | 33 ++++++++++--------- Python/ceval.c | 28 +++++++++------- Python/ceval_macros.h | 16 +++++---- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 33 ++++++++++--------- Python/instrumentation.c | 4 +-- 7 files changed, 64 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst new file mode 100644 index 00000000000000..f9b96788bfad94 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst @@ -0,0 +1 @@ +Improved the performance of :func:`sys.settrace` significantly diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e9e9425f826a2d..565379afc4b5a7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -143,22 +143,23 @@ dummy_func( tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - ERROR_IF(err, error); - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + ERROR_IF(err, error); + next_instr = this_instr; + DISPATCH(); } - this_instr->op.code = RESUME_CHECK; } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; } inst(RESUME_CHECK, (--)) { @@ -169,13 +170,13 @@ dummy_func( uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version); + DEOPT_IF(eval_breaker != version && tstate->tracing == 0); } inst(INSTRUMENTED_RESUME, (--)) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } diff --git a/Python/ceval.c b/Python/ceval.c index 06c136aeb252c9..41e9310938d826 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -800,17 +800,23 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int { _Py_CODEUNIT *prev = frame->instr_ptr; _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 085199416c1523..6ad41950ea3b7e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -339,12 +339,16 @@ do { \ // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) { \ - next_instr = (dest)+1; \ - goto error; \ + if (tstate->tracing) {\ + next_instr = dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ } \ } while (0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0b1d75985e1195..20fab8f4c61eb5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20,7 +20,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) goto deoptimize; + if (eval_breaker != version && tstate->tracing == 0) goto deoptimize; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8b90e448a7b8fa..bb26dac0e2be20 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3125,7 +3125,7 @@ INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } @@ -4780,22 +4780,23 @@ PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + DISPATCH(); } - this_instr->op.code = RESUME_CHECK; } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; DISPATCH(); } @@ -4811,7 +4812,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version && tstate->tracing == 0, RESUME); DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6f1bc2e0a107df..4b7d8b5a587504 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1156,15 +1156,13 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); + assert(tstate->tracing == 0); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - if (tstate->tracing) { - goto done; - } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); From f58f8cef7445ea04a69ba3e2848fffdb6b72df91 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 29 Feb 2024 02:51:59 +0900 Subject: [PATCH 37/82] gh-112075: Remove compiler warning from apple clang (gh-115855) --- Objects/dictobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 5ae4c3dbea2380..58fe973bc7a036 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5029,7 +5029,7 @@ dictiter_iternextkey(PyObject *self) PyObject *value; #ifdef Py_GIL_DISABLED - if (!dictiter_iternext_threadsafe(d, self, &value, NULL) == 0) { + if (dictiter_iternext_threadsafe(d, self, &value, NULL) < 0) { value = NULL; } #else @@ -5152,7 +5152,7 @@ dictiter_iternextvalue(PyObject *self) PyObject *value; #ifdef Py_GIL_DISABLED - if (!dictiter_iternext_threadsafe(d, self, NULL, &value) == 0) { + if (dictiter_iternext_threadsafe(d, self, NULL, &value) < 0) { value = NULL; } #else From e2a3e4b7488aff6fdc704a0f258bc315e96c1d6e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 28 Feb 2024 09:55:56 -0800 Subject: [PATCH 38/82] gh-115816: Improve internal symbols API in optimizer (#116028) - Any `sym_set_...` call that attempts to set conflicting information cause the symbol to become `bottom` (contradiction). - All `sym_is...` and similar calls return false or NULL for `bottom`. - Everything's tested. - The tests still pass with `PYTHONUOPSOPTIMIZE=1`. --- Include/internal/pycore_optimizer.h | 12 +- Python/optimizer_analysis.c | 2 + Python/optimizer_bytecodes.c | 2 + Python/optimizer_symbols.c | 242 +++++++++++++++++++++------- 4 files changed, 191 insertions(+), 67 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 425bd693fac53d..265eae4e290c38 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -27,12 +27,12 @@ extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; /* Symbols */ +/* See explanation in optimizer_symbols.c */ struct _Py_UopsSymbol { - int flags; - PyTypeObject *typ; - // constant propagated value (might be NULL) - PyObject *const_val; + int flags; // 0 bits: Top; 2 or more bits: Bottom + PyTypeObject *typ; // Borrowed reference + PyObject *const_val; // Owned reference (!) }; // Holds locals, stack, locals, stack ... co_consts (in that order) @@ -92,7 +92,9 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *con extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); extern void _Py_uop_sym_set_null(_Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *tp); +extern void _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern void _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index b29a00c941e996..8e408ffbb1c2b5 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -294,7 +294,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type #define sym_set_null _Py_uop_sym_set_null +#define sym_set_non_null _Py_uop_sym_set_non_null #define sym_set_type _Py_uop_sym_set_type +#define sym_set_const _Py_uop_sym_set_const #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 68737389c66b67..b65e90bf980e5a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -22,7 +22,9 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type #define sym_set_null _Py_uop_sym_set_null +#define sym_set_non_null _Py_uop_sym_set_non_null #define sym_set_type _Py_uop_sym_set_type +#define sym_set_const _Py_uop_sym_set_const #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 794d73733f85a7..158ee67d19f50e 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -10,11 +10,26 @@ #include #include +/* Symbols + ======= + + See the diagram at + https://github.com/faster-cpython/ideas/blob/main/3.13/redundancy_eliminator.md + + We represent the nodes in the diagram as follows + (the flag bits are only defined in optimizer_symbols.c): + - Top: no flag bits, typ and const_val are NULL. + - NULL: IS_NULL flag set, type and const_val NULL. + - Not NULL: NOT_NULL flag set, type and const_val NULL. + - None/not None: not used. (None could be represented as any other constant.) + - Known type: NOT_NULL flag set and typ set; const_val is NULL. + - Known constant: NOT_NULL flag set, type set, const_val set. + - Bottom: IS_NULL and NOT_NULL flags set, type and const_val NULL. + */ + // Flags for below. -#define KNOWN 1 << 0 -#define TRUE_CONST 1 << 1 -#define IS_NULL 1 << 2 -#define NOT_NULL 1 << 3 +#define IS_NULL 1 << 0 +#define NOT_NULL 1 << 1 #ifdef Py_DEBUG static inline int get_lltrace(void) { @@ -31,9 +46,8 @@ static inline int get_lltrace(void) { #define DPRINTF(level, ...) #endif -// Takes a borrowed reference to const_val, turns that into a strong reference. static _Py_UopsSymbol * -sym_new(_Py_UOpsContext *ctx, PyObject *const_val) +sym_new(_Py_UOpsContext *ctx) { _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { @@ -42,13 +56,9 @@ sym_new(_Py_UOpsContext *ctx, PyObject *const_val) return NULL; } ctx->t_arena.ty_curr_number++; - self->const_val = NULL; - self->typ = NULL; self->flags = 0; - - if (const_val != NULL) { - self->const_val = Py_NewRef(const_val); - } + self->typ = NULL; + self->const_val = NULL; return self; } @@ -59,59 +69,111 @@ sym_set_flag(_Py_UopsSymbol *sym, int flag) sym->flags |= flag; } +static inline void +sym_set_bottom(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL | NOT_NULL); + sym->typ = NULL; + Py_CLEAR(sym->const_val); +} + static inline bool -sym_has_flag(_Py_UopsSymbol *sym, int flag) +_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) { - return (sym->flags & flag) != 0; + if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { + assert(sym->flags == (IS_NULL | NOT_NULL)); + assert(sym->typ == NULL); + assert(sym->const_val == NULL); + return true; + } + return false; } bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) { - return (sym->flags & (IS_NULL | NOT_NULL)) == NOT_NULL; + return sym->flags == NOT_NULL; } bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym) { - return (sym->flags & (IS_NULL | NOT_NULL)) == IS_NULL; + return sym->flags == IS_NULL; } bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym) { - return (sym->flags & TRUE_CONST) != 0; + return sym->const_val != NULL; } PyObject * _Py_uop_sym_get_const(_Py_UopsSymbol *sym) { - assert(_Py_uop_sym_is_const(sym)); - assert(sym->const_val); return sym->const_val; } void -_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *tp) +_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { - assert(PyType_Check(tp)); - sym->typ = tp; - sym_set_flag(sym, KNOWN); + assert(typ != NULL && PyType_Check(typ)); + if (sym->flags & IS_NULL) { + sym_set_bottom(sym); + return; + } + if (sym->typ != NULL) { + if (sym->typ != typ) { + sym_set_bottom(sym); + } + return; + } + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; +} + +void +_Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) +{ + assert(const_val != NULL); + if (sym->flags & IS_NULL) { + sym_set_bottom(sym); + return; + } + PyTypeObject *typ = Py_TYPE(const_val); + if (sym->typ != NULL && sym->typ != typ) { + sym_set_bottom(sym); + return; + } + if (sym->const_val != NULL) { + if (sym->const_val != const_val) { + // TODO: What if they're equal? + sym_set_bottom(sym); + } + return; + } sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); } + void _Py_uop_sym_set_null(_Py_UopsSymbol *sym) { sym_set_flag(sym, IS_NULL); - sym_set_flag(sym, KNOWN); +} + +void +_Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, NOT_NULL); } _Py_UopsSymbol * _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) { - return sym_new(ctx, NULL); + return sym_new(ctx); } _Py_UopsSymbol * @@ -121,16 +183,14 @@ _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) if (res == NULL) { return NULL; } - sym_set_flag(res, KNOWN); sym_set_flag(res, NOT_NULL); return res; } _Py_UopsSymbol * -_Py_uop_sym_new_type(_Py_UOpsContext *ctx, - PyTypeObject *typ) +_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) { - _Py_UopsSymbol *res = sym_new(ctx,NULL); + _Py_UopsSymbol *res = sym_new(ctx); if (res == NULL) { return NULL; } @@ -138,20 +198,17 @@ _Py_uop_sym_new_type(_Py_UOpsContext *ctx, return res; } -// Takes a borrowed reference to const_val. +// Adds a new reference to const_val, owned by the symbol. _Py_UopsSymbol * _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) { assert(const_val != NULL); - _Py_UopsSymbol *temp = sym_new(ctx, const_val); - if (temp == NULL) { + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { return NULL; } - _Py_uop_sym_set_type(temp, Py_TYPE(const_val)); - sym_set_flag(temp, TRUE_CONST); - sym_set_flag(temp, KNOWN); - sym_set_flag(temp, NOT_NULL); - return temp; + _Py_uop_sym_set_const(res, const_val); + return res; } _Py_UopsSymbol * @@ -168,8 +225,8 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { - assert(typ == NULL || PyType_Check(typ)); - if (!sym_has_flag(sym, KNOWN)) { + assert(typ != NULL && PyType_Check(typ)); + if (_Py_uop_sym_is_bottom(sym)) { return false; } return sym->typ == typ; @@ -277,15 +334,14 @@ do { \ } \ } while (0) -/* static _Py_UopsSymbol * -make_contradiction(_Py_UOpsContext *ctx) +make_bottom(_Py_UOpsContext *ctx) { _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); _Py_uop_sym_set_null(sym); - _Py_uop_sym_set_type(sym, &PyLong_Type); + _Py_uop_sym_set_non_null(sym); return sym; -}*/ +} PyObject * _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) @@ -293,31 +349,93 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_UOpsContext context; _Py_UOpsContext *ctx = &context; _Py_uop_abstractcontext_init(ctx); + PyObject *val_42 = NULL; + PyObject *val_43 = NULL; - _Py_UopsSymbol *top = _Py_uop_sym_new_unknown(ctx); - if (top == NULL) { - return NULL; + // Use a single 'sym' variable so copy-pasting tests is easier. + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + if (sym == NULL) { + goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(top), "unknown is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(top), "unknown is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_const(top), "unknown is a constant"); - // TEST_PREDICATE(_Py_uop_sym_get_const(top) == NULL, "unknown as constant is not NULL"); - - // _Py_UopsSymbol *contradiction = make_contradiction(ctx); - // TEST_PREDICATE(_Py_uop_sym_is_null(contradiction), "contradiction is NULL is not true"); - // TEST_PREDICATE(_Py_uop_sym_is_not_null(contradiction), "contradiction is not NULL is not true"); - // TEST_PREDICATE(_Py_uop_sym_is_const(contradiction), "contradiction is a constant is not true"); - - _Py_UopsSymbol *int_type = _Py_uop_sym_new_type(ctx, &PyLong_Type); - TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "inconsistent type"); - _Py_uop_sym_set_type(int_type, &PyLong_Type); - TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "inconsistent type"); - _Py_uop_sym_set_type(int_type, &PyFloat_Type); - // TEST_PREDICATE(_Py_uop_sym_matches_type(int_type, &PyLong_Type), "(int and float) doesn't match int"); + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + + sym = make_bottom(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); + + _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + + _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + + val_42 = PyLong_FromLong(42); + assert(val_42 != NULL); + assert(_Py_IsImmortal(val_42)); + + val_43 = PyLong_FromLong(43); + assert(val_43 != NULL); + assert(_Py_IsImmortal(val_43)); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(sym, val_42); + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + + _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(sym, val_42); + _Py_uop_sym_set_const(sym, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); _Py_uop_abstractcontext_fini(ctx); + Py_DECREF(val_42); + Py_DECREF(val_43); Py_RETURN_NONE; + fail: _Py_uop_abstractcontext_fini(ctx); + Py_XDECREF(val_42); + Py_XDECREF(val_43); return NULL; } From 6c1c94dc517b77afcebb25436a4b7b0d13b6eb4d Mon Sep 17 00:00:00 2001 From: Kerim Kabirov <39376984+Privat33r-dev@users.noreply.github.com> Date: Wed, 28 Feb 2024 20:43:05 +0100 Subject: [PATCH 39/82] GH-115986 Reorder pprint docs and amend some references (#116019) Introduce a new subsubsection, 'Functions', for module level functions, and place it before the PrettyPrinter class reference. Also: - Fix pprint.pprint() references so they properly link to the module level function. - Add links to sys.stdout. --- Doc/library/pprint.rst | 181 +++++++++++++++++++++-------------------- 1 file changed, 92 insertions(+), 89 deletions(-) diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index e883acd67d6c72..2a2eb098646364 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -31,7 +31,93 @@ Dictionaries are sorted by key before the display is computed. .. versionchanged:: 3.10 Added support for pretty-printing :class:`dataclasses.dataclass`. -The :mod:`pprint` module defines one class: +.. _pprint-functions: + +Functions +--------- + +.. function:: pp(object, *args, sort_dicts=False, **kwargs) + + Prints the formatted representation of *object* followed by a newline. + If *sort_dicts* is false (the default), dictionaries will be displayed with + their keys in insertion order, otherwise the dict keys will be sorted. + *args* and *kwargs* will be passed to :func:`~pprint.pprint` as formatting + parameters. + + .. versionadded:: 3.8 + + +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Prints the formatted representation of *object* on *stream*, followed by a + newline. If *stream* is ``None``, :data:`sys.stdout` is used. This may be used + in the interactive interpreter instead of the :func:`print` function for + inspecting values (you can even reassign ``print = pprint.pprint`` for use + within a scope). + + The configuration parameters *stream*, *indent*, *width*, *depth*, + *compact*, *sort_dicts* and *underscore_numbers* are passed to the + :class:`PrettyPrinter` constructor and their meanings are as + described in its documentation above. + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pprint(stuff) + [, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + +.. function:: pformat(object, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Return the formatted representation of *object* as a string. *indent*, + *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + passed to the :class:`PrettyPrinter` constructor as formatting parameters + and their meanings are as described in its documentation above. + + +.. function:: isreadable(object) + + .. index:: pair: built-in function; eval + + Determine if the formatted representation of *object* is "readable", or can be + used to reconstruct the value using :func:`eval`. This always returns ``False`` + for recursive objects. + + >>> pprint.isreadable(stuff) + False + + +.. function:: isrecursive(object) + + Determine if *object* requires a recursive representation. This function is + subject to the same limitations as noted in :func:`saferepr` below and may raise an + :exc:`RecursionError` if it fails to detect a recursive object. + + +.. function:: saferepr(object) + + Return a string representation of *object*, protected against recursion in + some common data structures, namely instances of :class:`dict`, :class:`list` + and :class:`tuple` or subclasses whose ``__repr__`` has not been overridden. If the + representation of object exposes a recursive entry, the recursive reference + will be represented as ````. The + representation is not otherwise formatted. + + >>> pprint.saferepr(stuff) + "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" + +.. _prettyprinter-objects: + +PrettyPrinter Objects +--------------------- + +This module defines one class: .. First the implementation class: @@ -44,9 +130,9 @@ The :mod:`pprint` module defines one class: Construct a :class:`PrettyPrinter` instance. This constructor understands several keyword parameters. - *stream* (default ``sys.stdout``) is a :term:`file-like object` to + *stream* (default :data:`!sys.stdout`) is a :term:`file-like object` to which the output will be written by calling its :meth:`!write` method. - If both *stream* and ``sys.stdout`` are ``None``, then + If both *stream* and :data:`!sys.stdout` are ``None``, then :meth:`~PrettyPrinter.pprint` silently returns. Other values configure the manner in which nesting of complex data @@ -87,7 +173,7 @@ The :mod:`pprint` module defines one class: Added the *underscore_numbers* parameter. .. versionchanged:: 3.11 - No longer attempts to write to ``sys.stdout`` if it is ``None``. + No longer attempts to write to :data:`!sys.stdout` if it is ``None``. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -112,89 +198,6 @@ The :mod:`pprint` module defines one class: >>> pp.pprint(tup) ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) -.. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are - passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. - - -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`pprint` as formatting - parameters. - - .. versionadded:: 3.8 - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - -.. function:: isreadable(object) - - .. index:: pair: built-in function; eval - - Determine if the formatted representation of *object* is "readable", or can be - used to reconstruct the value using :func:`eval`. This always returns ``False`` - for recursive objects. - - >>> pprint.isreadable(stuff) - False - - -.. function:: isrecursive(object) - - Determine if *object* requires a recursive representation. This function is - subject to the same limitations as noted in :func:`saferepr` below and may raise an - :exc:`RecursionError` if it fails to detect a recursive object. - - -One more support function is also defined: - -.. function:: saferepr(object) - - Return a string representation of *object*, protected against recursion in - some common data structures, namely instances of :class:`dict`, :class:`list` - and :class:`tuple` or subclasses whose ``__repr__`` has not been overridden. If the - representation of object exposes a recursive entry, the recursive reference - will be represented as ````. The - representation is not otherwise formatted. - - >>> pprint.saferepr(stuff) - "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - - -.. _prettyprinter-objects: - -PrettyPrinter Objects ---------------------- :class:`PrettyPrinter` instances have the following methods: @@ -258,7 +261,7 @@ are converted to strings. The default implementation uses the internals of the Example ------- -To demonstrate several uses of the :func:`pprint` function and its parameters, +To demonstrate several uses of the :func:`~pprint.pprint` function and its parameters, let's fetch information about a project from `PyPI `_:: >>> import json @@ -267,7 +270,7 @@ let's fetch information about a project from `PyPI `_:: >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: ... project_info = json.load(resp)['info'] -In its basic form, :func:`pprint` shows the whole object:: +In its basic form, :func:`~pprint.pprint` shows the whole object:: >>> pprint.pprint(project_info) {'author': 'The Python Packaging Authority', From c43b26d02eaa103756c250e8d36829d388c5f3be Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 29 Feb 2024 04:15:52 +0800 Subject: [PATCH 40/82] gh-115197: Stop resolving host in urllib.request proxy bypass (GH-115210) Use of a proxy is intended to defer DNS for the hosts to the proxy itself, rather than a potential for information leak of the host doing DNS resolution itself for any reason. Proxy bypass lists are strictly name based. Most implementations of proxy support agree. --- Lib/test/test_urllib2.py | 29 ++++++- Lib/urllib/request.py | 77 +++++++++---------- ...-02-09-19-41-48.gh-issue-115197.20wkWH.rst | 2 + 3 files changed, 64 insertions(+), 44 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index fa528a675892b5..739c15df13de21 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -15,10 +15,11 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) from urllib.parse import urlparse @@ -1485,6 +1486,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; " + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index bca594420f6d9d..d22af6618d80f1 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -2563,6 +2563,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2579,20 +2580,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2615,6 +2613,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "" should bypass the proxy server for all intranet addresses + if test == '': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2713,7 +2736,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2723,40 +2746,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst new file mode 100644 index 00000000000000..e6ca3cc525d74a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst @@ -0,0 +1,2 @@ +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. From df5212df6c6f08308c68de4b3ed8a1b51ac6334b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 28 Feb 2024 15:37:59 -0500 Subject: [PATCH 41/82] gh-112529: Simplify PyObject_GC_IsTracked and PyObject_GC_IsFinalized (#114732) --- Modules/clinic/gcmodule.c.h | 40 ++++++++++++++++++++++++++++++++++++- Modules/gcmodule.c | 29 ++++++++++----------------- Python/gc_free_threading.c | 10 ++-------- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index d50d170589a2cd..9fff4da616ba00 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -469,6 +469,25 @@ PyDoc_STRVAR(gc_is_tracked__doc__, #define GC_IS_TRACKED_METHODDEF \ {"is_tracked", (PyCFunction)gc_is_tracked, METH_O, gc_is_tracked__doc__}, +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_tracked(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_tracked_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_is_finalized__doc__, "is_finalized($module, obj, /)\n" "--\n" @@ -478,6 +497,25 @@ PyDoc_STRVAR(gc_is_finalized__doc__, #define GC_IS_FINALIZED_METHODDEF \ {"is_finalized", (PyCFunction)gc_is_finalized, METH_O, gc_is_finalized__doc__}, +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_finalized(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_finalized_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_freeze__doc__, "freeze($module, /)\n" "--\n" @@ -547,4 +585,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=258f92524c1141fc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 961165e16a0fee..9807d2e7d48a36 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -383,7 +383,7 @@ gc_get_stats_impl(PyObject *module) /*[clinic input] -gc.is_tracked +gc.is_tracked -> bool obj: object / @@ -393,21 +393,15 @@ Returns true if the object is tracked by the garbage collector. Simple atomic objects will return false. [clinic start generated code]*/ -static PyObject * -gc_is_tracked(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=14f0103423b28e31 input=d83057f170ea2723]*/ +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=91c8d086b7f47a33 input=423b98ec680c3126]*/ { - PyObject *result; - - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) - result = Py_True; - else - result = Py_False; - return Py_NewRef(result); + return PyObject_GC_IsTracked(obj); } /*[clinic input] -gc.is_finalized +gc.is_finalized -> bool obj: object / @@ -415,14 +409,11 @@ gc.is_finalized Returns true if the object has been already finalized by the GC. [clinic start generated code]*/ -static PyObject * -gc_is_finalized(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=e1516ac119a918ed input=201d0c58f69ae390]*/ +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=401ff5d6fc660429 input=ca4d111c8f8c4e3a]*/ { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return PyObject_GC_IsFinalized(obj); } /*[clinic input] diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 14790899825de1..d4fb50106093ee 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1693,19 +1693,13 @@ PyObject_GC_Del(void *op) int PyObject_GC_IsTracked(PyObject* obj) { - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { - return 1; - } - return 0; + return _PyObject_GC_IS_TRACKED(obj); } int PyObject_GC_IsFinalized(PyObject *obj) { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - return 1; - } - return 0; + return _PyGC_FINALIZED(obj); } struct custom_visitor_args { From 75c6c05fea212330f4b0259602ffae1b2cb91be3 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 28 Feb 2024 15:50:09 -0500 Subject: [PATCH 42/82] gh-115891: Fix debug byte filling in free-threaded build (#116018) The previous code had two bugs. First, the debug offset in the mimalloc heap includes the two pymalloc debug words, but the pointer passed to fill_mem_debug does not include them. Second, the current object heap is correct source for allocations, but not deallocations. --- Objects/obmalloc.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 43427d4449eb1a..b2a2286ef22b66 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2460,14 +2460,23 @@ write_size_t(void *p, size_t n) } static void -fill_mem_debug(debug_alloc_api_t *api, void *data, int c, size_t nbytes) +fill_mem_debug(debug_alloc_api_t *api, void *data, int c, size_t nbytes, + bool is_alloc) { #ifdef Py_GIL_DISABLED if (api->api_id == 'o') { // Don't overwrite the first few bytes of a PyObject allocation in the // free-threaded build _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); - size_t debug_offset = tstate->mimalloc.current_object_heap->debug_offset; + size_t debug_offset; + if (is_alloc) { + debug_offset = tstate->mimalloc.current_object_heap->debug_offset; + } + else { + char *alloc = (char *)data - 2*SST; // start of the allocation + debug_offset = _mi_ptr_page(alloc)->debug_offset; + } + debug_offset -= 2*SST; // account for pymalloc extra bytes if (debug_offset < nbytes) { memset((char *)data + debug_offset, c, nbytes - debug_offset); } @@ -2553,7 +2562,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); if (nbytes > 0 && !use_calloc) { - fill_mem_debug(api, data, PYMEM_CLEANBYTE, nbytes); + fill_mem_debug(api, data, PYMEM_CLEANBYTE, nbytes, true); } /* at tail, write pad (SST bytes) and serialno (SST bytes) */ @@ -2603,7 +2612,7 @@ _PyMem_DebugRawFree(void *ctx, void *p) nbytes = read_size_t(q); nbytes += PYMEM_DEBUG_EXTRA_BYTES - 2*SST; memset(q, PYMEM_DEADBYTE, 2*SST); - fill_mem_debug(api, p, PYMEM_DEADBYTE, nbytes); + fill_mem_debug(api, p, PYMEM_DEADBYTE, nbytes, false); api->alloc.free(api->alloc.ctx, q); } From 3409bc29c9f06051c28ae0791155e3aebd76ff2d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 28 Feb 2024 14:38:01 -0800 Subject: [PATCH 43/82] gh-115859: Re-enable T2 optimizer pass by default (#116062) This undoes the *temporary* default disabling of the T2 optimizer pass in gh-115860. - Add a new test that reproduces Brandt's example from gh-115859; it indeed crashes before gh-116028 with PYTHONUOPSOPTIMIZE=1 - Re-enable the optimizer pass in T2, stop checking PYTHONUOPSOPTIMIZE - Rename the env var to disable T2 entirely to PYTHON_UOPS_OPTIMIZE (must be explicitly set to 0 to disable) - Fix skipIf conditions on tests in test_opt.py accordingly - Export sym_is_bottom() (for debugging) - Fix various things in the `_BINARY_OP_` specializations in the abstract interpreter: - DECREF(temp) - out-of-space check after sym_new_const() - add sym_matches_type() checks, so even if we somehow reach a binary op with symbolic constants of the wrong type on the stack we won't trigger the type assert --- Include/internal/pycore_optimizer.h | 2 ++ Lib/test/test_capi/test_opt.py | 21 +++++++++++++- Python/optimizer.c | 4 +-- Python/optimizer_analysis.c | 10 +++---- Python/optimizer_bytecodes.c | 43 +++++++++++++++++++++++------ Python/optimizer_cases.c.h | 42 ++++++++++++++++++++++------ Python/optimizer_symbols.c | 2 +- 7 files changed, 96 insertions(+), 28 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 265eae4e290c38..614850468ec1d3 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -95,6 +95,8 @@ extern void _Py_uop_sym_set_null(_Py_UopsSymbol *sym); extern void _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); extern void _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); + extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 25fc36dec93ddc..e1aef21b2c7644 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -210,6 +210,8 @@ def f(): exe = get_first_executor(f) self.assertIsNone(exe) + +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): def test_basic_loop(self): @@ -570,7 +572,7 @@ def testfunc(n): self.assertLessEqual(count, 2) -@unittest.skipIf(os.getenv("PYTHONUOPSOPTIMIZE", default=0) == 0, "Needs uop optimizer to run.") +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): def _run_with_optimizer(self, testfunc, arg): @@ -890,5 +892,22 @@ def testfunc(n): self.assertLessEqual(len(guard_both_float_count), 1) self.assertIn("_COMPARE_OP_STR", uops) + def test_type_inconsistency(self): + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + # Must be a real global else it won't be optimized to _LOAD_CONST_INLINE + global _test_global + _test_global = 0 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) + _test_global = 1.2 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_BOTH_INT", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + + if __name__ == "__main__": unittest.main() diff --git a/Python/optimizer.c b/Python/optimizer.c index c04ee17ee2171d..acd6d52c4a885f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1008,8 +1008,8 @@ uop_optimize( return err; } OPT_STAT_INC(traces_created); - char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); - if (uop_optimize == NULL || *uop_optimize > '0') { + char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); + if (env_var == NULL || *env_var == '\0' || *env_var > '0') { err = _Py_uop_analyze_and_optimize(frame, buffer, UOP_MAX_TRACE_LENGTH, curr_stackentries, &dependencies); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 8e408ffbb1c2b5..2a7ef4ec919eeb 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -297,6 +297,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_non_null _Py_uop_sym_set_non_null #define sym_set_type _Py_uop_sym_set_type #define sym_set_const _Py_uop_sym_set_const +#define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop @@ -510,12 +511,9 @@ _Py_uop_analyze_and_optimize( peephole_opt(frame, buffer, buffer_size); - char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); - if (uop_optimize != NULL && *uop_optimize > '0') { - err = optimize_uops( - (PyCodeObject *)frame->f_executable, buffer, - buffer_size, curr_stacklen, dependencies); - } + err = optimize_uops( + (PyCodeObject *)frame->f_executable, buffer, + buffer_size, curr_stacklen, dependencies); if (err == 0) { goto not_ready; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b65e90bf980e5a..928c22da16b999 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -25,6 +25,7 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_set_non_null _Py_uop_sym_set_non_null #define sym_set_type _Py_uop_sym_set_type #define sym_set_const _Py_uop_sym_set_const +#define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop @@ -107,7 +108,9 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), @@ -115,7 +118,9 @@ dummy_func(void) { if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } @@ -125,7 +130,9 @@ dummy_func(void) { } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), @@ -133,7 +140,9 @@ dummy_func(void) { if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } @@ -143,7 +152,9 @@ dummy_func(void) { } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), @@ -151,7 +162,9 @@ dummy_func(void) { if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } @@ -161,7 +174,9 @@ dummy_func(void) { } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { assert(PyFloat_CheckExact(sym_get_const(left))); assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( @@ -171,6 +186,8 @@ dummy_func(void) { goto error; } res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } @@ -180,7 +197,9 @@ dummy_func(void) { } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { assert(PyFloat_CheckExact(sym_get_const(left))); assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( @@ -190,6 +209,8 @@ dummy_func(void) { goto error; } res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } @@ -199,7 +220,9 @@ dummy_func(void) { } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { assert(PyFloat_CheckExact(sym_get_const(left))); assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( @@ -209,6 +232,8 @@ dummy_func(void) { goto error; } res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9e99c8395a405b..9b387c07850245 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -183,7 +183,9 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), @@ -191,7 +193,9 @@ if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } @@ -209,7 +213,9 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), @@ -217,7 +223,9 @@ if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } @@ -235,7 +243,9 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), @@ -243,7 +253,9 @@ if (temp == NULL) { goto error; } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp)); + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } @@ -275,7 +287,9 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { assert(PyFloat_CheckExact(sym_get_const(left))); assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( @@ -285,6 +299,8 @@ goto error; } res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } @@ -302,7 +318,9 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { assert(PyFloat_CheckExact(sym_get_const(left))); assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( @@ -312,6 +330,8 @@ goto error; } res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } @@ -329,7 +349,9 @@ _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { assert(PyFloat_CheckExact(sym_get_const(left))); assert(PyFloat_CheckExact(sym_get_const(right))); PyObject *temp = PyFloat_FromDouble( @@ -339,6 +361,8 @@ goto error; } res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 158ee67d19f50e..a529cc2f5cf215 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -77,7 +77,7 @@ sym_set_bottom(_Py_UopsSymbol *sym) Py_CLEAR(sym->const_val); } -static inline bool +bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) { if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { From 81c79961d2ee27dec90dbc0b72dfca7a5b27de7a Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Wed, 28 Feb 2024 14:53:19 -0800 Subject: [PATCH 44/82] gh-112075: Use relaxed stores for places where we may race with when reading lock-free (#115786) --- Objects/dictobject.c | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 58fe973bc7a036..5016e255f70ef9 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -250,6 +250,14 @@ load_keys_nentries(PyDictObject *mp) #endif +#define STORE_KEY(ep, key) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_key, key) +#define STORE_VALUE(ep, value) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, value) +#define STORE_SPLIT_VALUE(mp, idx, value) FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_values->values[idx], value) +#define STORE_HASH(ep, hash) FT_ATOMIC_STORE_SSIZE_RELAXED(ep->me_hash, hash) +#define STORE_KEYS_USABLE(keys, usable) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_usable, usable) +#define STORE_KEYS_NENTRIES(keys, nentries) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_nentries, nentries) +#define STORE_USED(mp, used) FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, used) + #define PERTURB_SHIFT 5 /* @@ -1621,7 +1629,6 @@ insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name) return ix; } - static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) @@ -1639,18 +1646,18 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep; ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - ep->me_value = value; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); } else { PyDictKeyEntry *ep; ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + STORE_HASH(ep, hash); } - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); + STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); assert(mp->ma_keys->dk_usable >= 0); return 0; } @@ -1682,7 +1689,7 @@ insert_split_dict(PyInterpreterState *interp, PyDictObject *mp, Py_ssize_t index = keys->dk_nentries; _PyDictValues_AddToInsertionOrder(mp->ma_values, index); assert (mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = value; + STORE_SPLIT_VALUE(mp, index, value); split_keys_entry_added(keys); assert(keys->dk_usable >= 0); @@ -2013,8 +2020,8 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } } - mp->ma_keys->dk_usable -= numentries; - mp->ma_keys->dk_nentries = numentries; + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - numentries); + STORE_KEYS_NENTRIES(mp->ma_keys, numentries); ASSERT_CONSISTENT(mp); return 0; } @@ -2507,15 +2514,15 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - ep->me_hash = 0; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); + STORE_HASH(ep, 0); } Py_DECREF(old_key); } @@ -4393,8 +4400,8 @@ dict_popitem_impl(PyDictObject *self) PyTuple_SET_ITEM(res, 0, key); PyTuple_SET_ITEM(res, 1, value); /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ - self->ma_keys->dk_nentries = i; - self->ma_used--; + STORE_KEYS_NENTRIES(self->ma_keys, i); + STORE_USED(self, self->ma_used - 1); self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; From 67c19e57b5c928278ebd191a545979ce786f06b3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 28 Feb 2024 17:04:56 -0600 Subject: [PATCH 45/82] Improve all_equal() recipe (gh-116081) Replace conjuction of next() calls with simpler len()/take() logic. Add key function. --- Doc/library/itertools.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 42e70404b306b0..4e731fefe8908d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -863,10 +863,9 @@ which incur interpreter overhead. "Given a predicate that returns True or False, count the True results." return sum(map(pred, iterable)) - def all_equal(iterable): + def all_equal(iterable, key=None): "Returns True if all the elements are equal to each other." - g = groupby(iterable) - return next(g, True) and not next(g, False) + return len(take(2, groupby(iterable, key))) <= 1 def first_true(iterable, default=False, pred=None): """Returns the first true value in the iterable. @@ -1225,6 +1224,8 @@ The following recipes have a more mathematical flavor: >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] + [True, True, True, False, False] >>> quantify(range(99), lambda x: x%2==0) 50 From e80abd57a82ea1beae0a82423d45c6eb8c5c5c74 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 28 Feb 2024 16:08:08 -0700 Subject: [PATCH 46/82] gh-76785: Update test.support.interpreters to Align With PEP 734 (gh-115566) This brings the code under test.support.interpreters, and the corresponding extension modules, in line with recent updates to PEP 734. (Note: PEP 734 has not been accepted at this time. However, we are using an internal copy of the implementation in the test suite to exercise the existing subinterpreters feature.) --- Lib/test/support/interpreters/__init__.py | 51 +++- Lib/test/support/interpreters/queues.py | 74 ++++- Lib/test/test_interpreters/test_api.py | 291 +++++++++++++++---- Lib/test/test_interpreters/test_channels.py | 4 +- Lib/test/test_interpreters/test_lifecycle.py | 2 +- Lib/test/test_interpreters/test_queues.py | 151 +++++++--- Lib/test/test_interpreters/utils.py | 19 +- Lib/test/test_sys.py | 4 +- Lib/test/test_threading.py | 2 +- Modules/_xxinterpqueuesmodule.c | 125 +++++--- Modules/_xxsubinterpretersmodule.c | 58 ++++ 11 files changed, 624 insertions(+), 157 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 15a908e9663593..d02ffbae1113c0 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -6,7 +6,7 @@ # aliases: from _xxsubinterpreters import ( - InterpreterError, InterpreterNotFoundError, + InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) @@ -14,7 +14,8 @@ __all__ = [ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', 'Interpreter', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', ] @@ -42,7 +43,11 @@ def __getattr__(name): {formatted} """.strip() -class ExecFailure(RuntimeError): +class ExecutionFailed(RuntimeError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ def __init__(self, excinfo): msg = excinfo.formatted @@ -157,7 +162,7 @@ def prepare_main(self, ns=None, /, **kwargs): ns = dict(ns, **kwargs) if ns is not None else kwargs _interpreters.set___main___attrs(self._id, ns) - def exec_sync(self, code, /): + def exec(self, code, /): """Run the given source code in the interpreter. This is essentially the same as calling the builtin "exec" @@ -166,10 +171,10 @@ def exec_sync(self, code, /): There is no return value. - If the code raises an unhandled exception then an ExecFailure - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. This blocks the current Python thread until done. During that time, the previous interpreter is allowed to run @@ -177,11 +182,35 @@ def exec_sync(self, code, /): """ excinfo = _interpreters.exec(self._id, code) if excinfo is not None: - raise ExecFailure(excinfo) + raise ExecutionFailed(excinfo) + + def call(self, callable, /): + """Call the object in the interpreter with given args/kwargs. + + Only functions that take no arguments and have no closure + are supported. - def run(self, code, /): + The return value is discarded. + + If the callable raises an exception then the error display + (including full traceback) is send back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + # XXX Support args and kwargs. + # XXX Support arbitrary callables. + # XXX Support returning the return value (e.g. via pickle). + excinfo = _interpreters.call(self._id, callable) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call_in_thread(self, callable, /): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ def task(): - self.exec_sync(code) + self.call(callable) t = threading.Thread(target=task) t.start() return t diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index aead0c40ca9667..2cc616be337a50 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -1,5 +1,6 @@ """Cross-interpreter Queues High Level Module.""" +import pickle import queue import time import weakref @@ -31,20 +32,26 @@ class QueueFull(_queues.QueueFull, queue.Full): """ -def create(maxsize=0): +_SHARED_ONLY = 0 +_PICKLED = 1 + +def create(maxsize=0, *, syncobj=False): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. + + "syncobj" sets the default for Queue.put() + and Queue.put_nowait(). """ - qid = _queues.create(maxsize) - return Queue(qid) + fmt = _SHARED_ONLY if syncobj else _PICKLED + qid = _queues.create(maxsize, fmt) + return Queue(qid, _fmt=fmt) def list_all(): """Return a list of all open queues.""" - return [Queue(qid) - for qid in _queues.list_all()] - + return [Queue(qid, _fmt=fmt) + for qid, fmt in _queues.list_all()] _known_queues = weakref.WeakValueDictionary() @@ -52,17 +59,20 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /): + def __new__(cls, id, /, *, _fmt=None): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') + if _fmt is None: + _fmt = _queues.get_default_fmt(id) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id + self._fmt = _fmt _known_queues[id] = self _queues.bind(id) return self @@ -105,20 +115,50 @@ def qsize(self): return _queues.get_count(self._id) def put(self, obj, timeout=None, *, + syncobj=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. This blocks while the queue is full. + + If "syncobj" is None (the default) then it uses the + queue's default, set with create_queue().. + + If "syncobj" is false then all objects are supported, + at the expense of worse performance. + + If "syncobj" is true then the object must be "shareable". + Examples of "shareable" objects include the builtin singletons, + str, and memoryview. One benefit is that such objects are + passed through the queue efficiently. + + The key difference, though, is conceptual: the corresponding + object returned from Queue.get() will be strictly equivalent + to the given obj. In other words, the two objects will be + effectively indistinguishable from each other, even if the + object is mutable. The received object may actually be the + same object, or a copy (immutable values only), or a proxy. + Regardless, the received object should be treated as though + the original has been shared directly, whether or not it + actually is. That's a slightly different and stronger promise + than just (initial) equality, which is all "syncobj=False" + can promise. """ + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout + if fmt is _PICKLED: + obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj) + _queues.put(self._id, obj, fmt) except _queues.QueueFull as exc: if timeout is not None and time.time() >= end: exc.__class__ = QueueFull @@ -127,9 +167,15 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj): + def put_nowait(self, obj, *, syncobj=None): + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if fmt is _PICKLED: + obj = pickle.dumps(obj) try: - return _queues.put(self._id, obj) + _queues.put(self._id, obj, fmt) except _queues.QueueFull as exc: exc.__class__ = QueueFull raise # re-raise @@ -148,12 +194,18 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - return _queues.get(self._id) + obj, fmt = _queues.get(self._id) except _queues.QueueEmpty as exc: if timeout is not None and time.time() >= end: exc.__class__ = QueueEmpty raise # re-raise time.sleep(_delay) + else: + break + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY return obj def get_nowait(self): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index aefd326977095f..363143fa810f35 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -280,7 +280,7 @@ def test_subinterpreter(self): def test_finished(self): r, w = self.pipe() interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os os.write({w}, b'x') """) @@ -312,7 +312,7 @@ def test_with_only_background_threads(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -326,7 +326,7 @@ def task(): self.assertFalse(interp.is_running()) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) @@ -393,7 +393,7 @@ def test_from_sibling(self): interp2 = interpreters.create() self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) - interp1.exec_sync(dedent(f""" + interp1.exec(dedent(f""" from test.support import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() @@ -427,7 +427,7 @@ def test_subthreads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time @@ -503,27 +503,27 @@ def test_not_shareable(self): interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) # Make sure neither was actually bound. - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(foo)') - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(spam)') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('print(foo)') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('print(spam)') -class TestInterpreterExecSync(TestBase): +class TestInterpreterExec(TestBase): def test_success(self): interp = interpreters.create() script, file = _captured_script('print("it worked!", end="")') with file: - interp.exec_sync(script) + interp.exec(script) out = file.read() self.assertEqual(out, 'it worked!') def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('raise Exception') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('raise Exception') def test_display_preserved_exception(self): tempdir = self.temp_dir() @@ -542,21 +542,21 @@ def script(): spam.eggs() interp = interpreters.create() - interp.exec_sync(script) + interp.exec(script) """) stdout, stderr = self.assert_python_failure(scriptfile) self.maxDiff = None - interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l) - # File "{interpreters.__file__}", line 179, in exec_sync + interpmod_line, = (l for l in stderr.splitlines() if ' exec' in l) + # File "{interpreters.__file__}", line 179, in exec self.assertEqual(stderr, dedent(f"""\ Traceback (most recent call last): File "{scriptfile}", line 9, in - interp.exec_sync(script) - ~~~~~~~~~~~~~~~~^^^^^^^^ + interp.exec(script) + ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} - raise ExecFailure(excinfo) - test.support.interpreters.ExecFailure: RuntimeError: uh-oh! + raise ExecutionFailed(excinfo) + test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -578,7 +578,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interp.exec_sync(script) + interp.exec(script) t = threading.Thread(target=f) t.start() @@ -604,7 +604,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interp.exec_sync(script) + interp.exec(script) file.seek(0) content = file.read() @@ -615,17 +615,17 @@ def test_already_running(self): interp = interpreters.create() with _running(interp): with self.assertRaises(RuntimeError): - interp.exec_sync('print("spam")') + interp.exec('print("spam")') def test_bad_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(10) + interp.exec(10) def test_bytes_for_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(b'print("spam")') + interp.exec(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -636,7 +636,7 @@ def test_with_background_threads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -648,46 +648,229 @@ def task(): t.start() os.write({w_interp}, {RAN!r}) """) - interp.exec_sync(f"""if True: + interp.exec(f"""if True: os.write({w_interp}, {RAN!r}) """) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) # test_xxsubinterpreters covers the remaining - # Interpreter.exec_sync() behavior. + # Interpreter.exec() behavior. -class TestInterpreterRun(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - t = interp.run(script) +def call_func_noop(): + pass + + +def call_func_return_shareable(): + return (1, None) + + +def call_func_return_not_shareable(): + return [1, 2, 3] + + +def call_func_failure(): + raise Exception('spam!') + + +def call_func_ident(value): + return value + + +def get_call_func_closure(value): + def call_func_closure(): + return value + return call_func_closure + + +class Spam: + + @staticmethod + def noop(): + pass + + @classmethod + def from_values(cls, *values): + return cls(values) + + def __init__(self, value): + self.value = value + + def __call__(self, *args, **kwargs): + return (self.value, args, kwargs) + + def __eq__(self, other): + if not isinstance(other, Spam): + return NotImplemented + return self.value == other.value + + def run(self, *args, **kwargs): + return (self.value, args, kwargs) + + +def call_func_complex(op, /, value=None, *args, exc=None, **kwargs): + if exc is not None: + raise exc + if op == '': + raise ValueError('missing op') + elif op == 'ident': + if args or kwargs: + raise Exception((args, kwargs)) + return value + elif op == 'full-ident': + return (value, args, kwargs) + elif op == 'globals': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return __name__ + elif op == 'interpid': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return interpreters.get_current().id + elif op == 'closure': + if args or kwargs: + raise Exception((args, kwargs)) + return get_call_func_closure(value) + elif op == 'custom': + if args or kwargs: + raise Exception((args, kwargs)) + return Spam(value) + elif op == 'custom-inner': + if args or kwargs: + raise Exception((args, kwargs)) + class Eggs(Spam): + pass + return Eggs(value) + elif not isinstance(op, str): + raise TypeError(op) + else: + raise NotImplementedError(op) + + +class TestInterpreterCall(TestBase): + + # signature + # - blank + # - args + # - kwargs + # - args, kwargs + # return + # - nothing (None) + # - simple + # - closure + # - custom + # ops: + # - do nothing + # - fail + # - echo + # - do complex, relative to interpreter + # scope + # - global func + # - local closure + # - returned closure + # - callable type instance + # - type + # - classmethod + # - staticmethod + # - instance method + # exception + # - builtin + # - custom + # - preserves info (e.g. SyntaxError) + # - matching error display + + def test_call(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable) + self.assertIs(res, None) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + with self.assertRaises(Exception): + if args or kwargs: + raise Exception((args, kwargs)) + interp.call(callable) + + with self.assertRaises(interpreters.ExecutionFailed): + interp.call(call_func_failure) + + def test_call_in_thread(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + if args or kwargs: + continue + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNotNone(ctx.caught) + + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(call_func_failure) t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_failure(self): - caught = False - def excepthook(args): - nonlocal caught - caught = True - threading.excepthook = excepthook - try: - interp = interpreters.create() - t = interp.run('raise Exception') - t.join() - - self.assertTrue(caught) - except BaseException: - threading.excepthook = threading.__excepthook__ + self.assertIsNotNone(ctx.caught) class TestIsShareable(TestBase): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 3c3e18832d4168..07e503837bcf75 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -120,7 +120,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' @@ -193,7 +193,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index c2917d839904f9..67b6f439c3191f 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -124,7 +124,7 @@ def test_sys_path_0(self): orig = sys.path[0] interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import json import sys print(json.dumps({{ diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 2a8ca99c1f6e3f..65b5435fb00b04 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -51,20 +51,20 @@ def test_shareable(self): queue1 = queues.create() interp = interpreters.create() - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" from test.support.interpreters import queues queue1 = queues.Queue({queue1.id}) """)); with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2) + queue1.put(queue2, syncobj=True) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4) + queue1.put(queue4, syncobj=True) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -75,7 +75,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5) + queue1.put(queue5, syncobj=True) print(queue5.id) """)) qid = int(out) @@ -118,7 +118,7 @@ class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None) + queue.put(None, syncobj=True) during = queue.empty() queue.get() after = queue.empty() @@ -133,7 +133,7 @@ def test_full(self): queue = queues.create(3) for _ in range(3): actual.append(queue.full()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.full()) for _ in range(3): queue.get() @@ -147,16 +147,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -165,30 +165,81 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - queue = queues.create() - for i in range(20): - queue.put(i) - actual = [queue.get() for _ in range(20)] + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create() + for i in range(20): + queue.put(i, **kwds) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - queue = queues.create(2) - queue.put(None) - queue.put(None) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1) - queue.get() - queue.put(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put(None, **kwds) + queue.put(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1, **kwds) + queue.get() + queue.put(None, **kwds) def test_put_nowait(self): - queue = queues.create(2) - queue.put_nowait(None) - queue.put_nowait(None) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None) - queue.get() - queue.put_nowait(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put_nowait(None, **kwds) + queue.put_nowait(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None, **kwds) + queue.get() + queue.put_nowait(None, **kwds) + + def test_put_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + ]: + with self.subTest(repr(obj)): + queue = queues.create() + queue.put(obj, syncobj=True) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + for obj in [ + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj, syncobj=True) + + def test_put_not_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + # not shareable + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + queue.put(obj, syncobj=False) + obj2 = queue.get() + self.assertEqual(obj2, obj) def test_get_timeout(self): queue = queues.create() @@ -200,13 +251,41 @@ def test_get_nowait(self): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() + def test_put_get_default_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=True) + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj) + + def test_put_get_default_not_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=False) + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + queue.put(obj) + obj2 = queue.get() + self.assertEqual(obj, obj2) + self.assertIsNot(obj, obj2) + def test_put_get_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import queues queue = queues.create() orig = b'spam' - queue.put(orig) + queue.put(orig, syncobj=True) obj = queue.get() assert obj == orig, 'expected: obj == orig' assert obj is not orig, 'expected: obj is not orig' @@ -219,7 +298,7 @@ def test_put_get_different_interpreters(self): self.assertEqual(len(queues.list_all()), 2) obj1 = b'spam' - queue1.put(obj1) + queue1.put(obj1, syncobj=True) out = _run_output( interp, @@ -236,7 +315,7 @@ def test_put_get_different_interpreters(self): obj2 = b'eggs' print(id(obj2)) assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2) + queue2.put(obj2, syncobj=True) assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' """)) self.assertEqual(len(queues.list_all()), 2) @@ -258,8 +337,8 @@ def test_put_cleared_with_subinterpreter(self): queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1) - queue.put(obj2) + queue.put(obj1, syncobj=True) + queue.put(obj2, syncobj=True) """)) self.assertEqual(queue.qsize(), 2) @@ -281,12 +360,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj) + queue2.put(obj, syncobj=True) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig) + queue1.put(orig, syncobj=True) obj = queue2.get() t.join() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 3a37ed09dd8943..973d05d4f96dcb 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -4,8 +4,9 @@ import subprocess import sys import tempfile -import threading from textwrap import dedent +import threading +import types import unittest from test import support @@ -41,7 +42,7 @@ def _run_output(interp, request, init=None): with rpipe: if init: interp.prepare_main(init) - interp.exec_sync(script) + interp.exec(script) return rpipe.read() @@ -49,7 +50,7 @@ def _run_output(interp, request, init=None): def _running(interp): r, w = os.pipe() def run(): - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" # wait for "signal" with open({r}) as rpipe: rpipe.read() @@ -84,6 +85,18 @@ def temp_dir(self): self.addCleanup(lambda: os_helper.rmtree(tempdir)) return tempdir + @contextlib.contextmanager + def captured_thread_exception(self): + ctx = types.SimpleNamespace(caught=None) + def excepthook(args): + ctx.caught = args + orig_excepthook = threading.excepthook + threading.excepthook = excepthook + try: + yield ctx + finally: + threading.excepthook = orig_excepthook + def make_script(self, filename, dirname=None, text=None): if text: text = dedent(text) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 71671a5a984256..38dcabd84d8170 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -729,7 +729,7 @@ def test_subinterp_intern_dynamically_allocated(self): self.assertIs(t, s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) != {id(s)}, (id(t), {id(s)}) @@ -744,7 +744,7 @@ def test_subinterp_intern_statically_allocated(self): t = sys.intern(s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) == {id(t)}, (id(t), {id(t)}) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 1ab223b81e939e..3b5c37c948c8c3 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1478,7 +1478,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 7d8c67f49fefb8..715bb766cac624 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -294,6 +294,8 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) case ERR_QUEUES_ALLOC: PyErr_NoMemory(); break; + case -1: + return -1; default: state = get_module_state(mod); assert(state->QueueError != NULL); @@ -320,14 +322,17 @@ struct _queueitem; typedef struct _queueitem { _PyCrossInterpreterData *data; + int fmt; struct _queueitem *next; } _queueitem; static void -_queueitem_init(_queueitem *item, _PyCrossInterpreterData *data) +_queueitem_init(_queueitem *item, + _PyCrossInterpreterData *data, int fmt) { *item = (_queueitem){ .data = data, + .fmt = fmt, }; } @@ -344,14 +349,14 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(_PyCrossInterpreterData *data) +_queueitem_new(_PyCrossInterpreterData *data, int fmt) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, data); + _queueitem_init(item, data, fmt); return item; } @@ -373,9 +378,11 @@ _queueitem_free_all(_queueitem *item) } static void -_queueitem_popped(_queueitem *item, _PyCrossInterpreterData **p_data) +_queueitem_popped(_queueitem *item, + _PyCrossInterpreterData **p_data, int *p_fmt) { *p_data = item->data; + *p_fmt = item->fmt; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; _queueitem_free(item); @@ -393,10 +400,11 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; + int fmt; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize) +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { @@ -408,6 +416,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize) .items = { .maxsize = maxsize, }, + .fmt = fmt, }; return 0; } @@ -486,7 +495,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, _PyCrossInterpreterData *data) +_queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) { int err = _queue_lock(queue); if (err < 0) { @@ -502,7 +511,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(data); + _queueitem *item = _queueitem_new(data, fmt); if (item == NULL) { _queue_unlock(queue); return -1; @@ -522,7 +531,8 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) } static int -_queue_next(_queue *queue, _PyCrossInterpreterData **p_data) +_queue_next(_queue *queue, + _PyCrossInterpreterData **p_data, int *p_fmt) { int err = _queue_lock(queue); if (err < 0) { @@ -541,7 +551,7 @@ _queue_next(_queue *queue, _PyCrossInterpreterData **p_data) } queue->items.count -= 1; - _queueitem_popped(item, p_data); + _queueitem_popped(item, p_data, p_fmt); _queue_unlock(queue); return 0; @@ -843,18 +853,26 @@ _queues_decref(_queues *queues, int64_t qid) PyThread_release_lock(queues->mutex); } -static int64_t * +struct queue_id_and_fmt { + int64_t id; + int fmt; +}; + +static struct queue_id_and_fmt * _queues_list_all(_queues *queues, int64_t *count) { - int64_t *qids = NULL; + struct queue_id_and_fmt *qids = NULL; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(queues->count)); + struct queue_id_and_fmt *ids = PyMem_NEW(struct queue_id_and_fmt, + (Py_ssize_t)(queues->count)); if (ids == NULL) { goto done; } _queueref *ref = queues->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->qid; + ids[i].id = ref->qid; + assert(ref->queue != NULL); + ids[i].fmt = ref->queue->fmt; } *count = queues->count; @@ -890,13 +908,13 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize) +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize); + int err = _queue_init(queue, maxsize, fmt); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -925,7 +943,7 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj) +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) { // Look up the queue. _queue *queue = NULL; @@ -948,7 +966,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) } // Add the data to the queue. - int res = _queue_add(queue, data); + int res = _queue_add(queue, data, fmt); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: @@ -963,7 +981,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) // Pop the next object off the queue. Fail if empty. // XXX Support a "wait" mutex? static int -queue_get(_queues *queues, int64_t qid, PyObject **res) +queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) { int err; *res = NULL; @@ -979,7 +997,7 @@ queue_get(_queues *queues, int64_t qid, PyObject **res) // Pop off the next item from the queue. _PyCrossInterpreterData *data = NULL; - err = _queue_next(queue, &data); + err = _queue_next(queue, &data, p_fmt); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1267,14 +1285,15 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", NULL}; - Py_ssize_t maxsize = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n:create", kwlist, - &maxsize)) { + static char *kwlist[] = {"maxsize", "fmt", NULL}; + Py_ssize_t maxsize; + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "ni:create", kwlist, + &maxsize, &fmt)) { return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize); + int64_t qid = queue_create(&_globals.queues, maxsize, fmt); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1329,7 +1348,7 @@ static PyObject * queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *qids = _queues_list_all(&_globals.queues, &count); + struct queue_id_and_fmt *qids = _queues_list_all(&_globals.queues, &count); if (qids == NULL) { if (count == 0) { return PyList_New(0); @@ -1340,14 +1359,14 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } - int64_t *cur = qids; + struct queue_id_and_fmt *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *qidobj = PyLong_FromLongLong(*cur); - if (qidobj == NULL) { + PyObject *item = Py_BuildValue("Li", cur->id, cur->fmt); + if (item == NULL) { Py_SETREF(ids, NULL); break; } - PyList_SET_ITEM(ids, (Py_ssize_t)i, qidobj); + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -1363,17 +1382,18 @@ Return the list of IDs for all queues."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", NULL}; + static char *kwlist[] = {"qid", "obj", "fmt", NULL}; qidarg_converter_data qidarg; PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:put", kwlist, - qidarg_converter, &qidarg, &obj)) { + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oi:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt)) { return NULL; } int64_t qid = qidarg.id; /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj); + int err = queue_put(&_globals.queues, qid, obj, fmt); if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1382,7 +1402,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj)\n\ +"put(qid, obj, sharedonly=False)\n\ \n\ Add the object's data to the queue."); @@ -1399,7 +1419,8 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) int64_t qid = qidarg.id; PyObject *obj = NULL; - int err = queue_get(&_globals.queues, qid, &obj); + int fmt; + int err = queue_get(&_globals.queues, qid, &obj, &fmt); if (err == ERR_QUEUE_EMPTY && dflt != NULL) { assert(obj == NULL); obj = Py_NewRef(dflt); @@ -1407,7 +1428,10 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) else if (handle_queue_error(err, self, qid)) { return NULL; } - return obj; + + PyObject *res = Py_BuildValue("Oi", obj, fmt); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(queuesmod_get_doc, @@ -1499,6 +1523,33 @@ PyDoc_STRVAR(queuesmod_get_maxsize_doc, \n\ Return the maximum number of items in the queue."); +static PyObject * +queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_default_fmt", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + _queue *queue = NULL; + int err = _queues_lookup(&_globals.queues, qid, &queue); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + int fmt = queue->fmt; + _queue_unmark_waiter(queue, _globals.queues.mutex); + return PyLong_FromLong(fmt); +} + +PyDoc_STRVAR(queuesmod_get_default_fmt_doc, +"get_default_fmt(qid)\n\ +\n\ +Return the default format to use for the queue."); + static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1593,6 +1644,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"get_default_fmt", _PyCFunction_CAST(queuesmod_get_default_fmt), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_default_fmt_doc}, {"is_full", _PyCFunction_CAST(queuesmod_is_full), METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index b4004d165078f7..28c2f9c08bc0da 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -902,6 +902,56 @@ The code/function must not take any arguments or be a closure\n\ If a function is provided, its code object is used and all its state\n\ is ignored, including its __globals__ dict."); +static PyObject * +interp_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "callable", "args", "kwargs", NULL}; + PyObject *id, *callable; + PyObject *args_obj = NULL; + PyObject *kwargs_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|OO:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj)) { + return NULL; + } + + if (args_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected args"); + return NULL; + } + if (kwargs_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + return NULL; + } + + PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", + "argument 2", "a function"); + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, id, code, NULL, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(call_doc, +"call(id, callable, args=None, kwargs=None)\n\ +\n\ +Call the provided object in the identified interpreter.\n\ +Pass the given args and kwargs, if possible.\n\ +\n\ +\"callable\" may be a plain function with no free vars that takes\n\ +no arguments.\n\ +\n\ +The function's code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1085,6 +1135,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, is_running_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, + {"call", _PyCFunction_CAST(interp_call), + METH_VARARGS | METH_KEYWORDS, call_doc}, {"run_string", _PyCFunction_CAST(interp_run_string), METH_VARARGS | METH_KEYWORDS, run_string_doc}, {"run_func", _PyCFunction_CAST(interp_run_func), @@ -1113,6 +1165,7 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { + PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); // exceptions @@ -1122,6 +1175,11 @@ module_exec(PyObject *mod) if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { goto error; } + PyObject *PyExc_NotShareableError = \ + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + goto error; + } if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { goto error; From f484a2a7486d0b4c7c11901f6c668eb23b74e81f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 28 Feb 2024 17:11:05 -0600 Subject: [PATCH 47/82] Update an out-of-date example in the itertools recipe intro (gh-116082) --- Doc/library/itertools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 4e731fefe8908d..c26f6c89b4920a 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -778,7 +778,7 @@ The primary purpose of the itertools recipes is educational. The recipes show various ways of thinking about individual tools — for example, that ``chain.from_iterable`` is related to the concept of flattening. The recipes also give ideas about ways that the tools can be combined — for example, how -``compress()`` and ``range()`` can work together. The recipes also show patterns +``starmap()`` and ``repeat()`` can work together. The recipes also show patterns for using itertools with the :mod:`operator` and :mod:`collections` modules as well as with the built-in itertools such as ``map()``, ``filter()``, ``reversed()``, and ``enumerate()``. From 4d1d35b906010c6db15f54443a9701c20af1db2d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Feb 2024 00:16:01 +0100 Subject: [PATCH 48/82] gh-116075: Skip test_external_inspection on qemu in JIT CI (#116076) --- .github/workflows/jit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 21d4603b8679ea..43d0b2c1b4016c 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -70,13 +70,13 @@ jobs: runner: ubuntu-latest compiler: gcc # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 runner: ubuntu-latest compiler: clang # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection env: CC: ${{ matrix.compiler }} steps: From 3ea78fd5bc93fc339ef743e6a5dfde35f04d972e Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Wed, 28 Feb 2024 15:17:49 -0800 Subject: [PATCH 49/82] gh-115821: [Enum] better error message for calling super().__new__() (GH-116063) docs now state to not call super().__new__ if super().__new__ is called, a better error message is now used --- Doc/library/enum.rst | 3 +++ Lib/enum.py | 5 +++++ Lib/test/test_enum.py | 13 ++++++++++++- .../2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst | 2 ++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 30d80ce8d488cc..6e7de004cd52a1 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -400,6 +400,9 @@ Data Types results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + ..note:: When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. + .. method:: Enum.__repr__(self) Returns the string used for *repr()* calls. By default, returns the diff --git a/Lib/enum.py b/Lib/enum.py index d10b99615981ba..22963cca4466f2 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -547,7 +547,10 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_inverted_'] = None try: exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) except Exception as e: # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." # is tacked on to the error instead of raising a RuntimeError @@ -1155,6 +1158,8 @@ def __new__(cls, value): # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index cf3e042de1a4b4..27f8bbaf952afc 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -447,7 +447,7 @@ def spam(cls): def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -3409,6 +3409,17 @@ def __new__(cls, int_value, *value_aliases): self.assertIs(Types(2), Types.NetList) self.assertIs(Types('nl'), Types.NetList) + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" diff --git a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst new file mode 100644 index 00000000000000..7512a09a37cd46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst @@ -0,0 +1,2 @@ +[Enum] Improve error message when calling super().__new__() in custom +__new__. From 479ac5ce8a311c9a5830b96e972478867fcbce61 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 28 Feb 2024 15:56:58 -0800 Subject: [PATCH 50/82] gh-115859: Fix test_type_inconsistency() when run multiple times (#116079) This should fix the refleaks bots. (See https://github.com/python/cpython/pull/116062#issuecomment-1970038174 .) --- Lib/test/test_capi/test_opt.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index e1aef21b2c7644..a43726f05a448d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -893,9 +893,13 @@ def testfunc(n): self.assertIn("_COMPARE_OP_STR", uops) def test_type_inconsistency(self): - def testfunc(n): - for i in range(n): - x = _test_global + _test_global + ns = {} + exec(textwrap.dedent(""" + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + """), globals(), ns) + testfunc = ns['testfunc'] # Must be a real global else it won't be optimized to _LOAD_CONST_INLINE global _test_global _test_global = 0 From 86e5e063aba76a7f4fc58f7d06b17b0a4730fd8e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 28 Feb 2024 16:05:53 -0800 Subject: [PATCH 51/82] gh-115816: Generate calls to sym_new_const() etc. without _Py_uop prefix (#116077) This was left behind by GH-115987. Basically a lot of diffs like this: ``` - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); ``` --- Lib/test/test_generated_cases.py | 4 +- Python/optimizer_cases.c.h | 196 +++++++++---------- Tools/cases_generator/optimizer_generator.py | 8 +- 3 files changed, 104 insertions(+), 104 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 18bf8ab29148c4..6fcb5d58dd7f34 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -900,7 +900,7 @@ def test_overridden_abstract_args(self): case OP2: { _Py_UopsSymbol *out; - out = _Py_uop_sym_new_unknown(ctx); + out = sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; @@ -925,7 +925,7 @@ def test_no_overridden_case(self): output = """ case OP: { _Py_UopsSymbol *out; - out = _Py_uop_sym_new_unknown(ctx); + out = sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9b387c07850245..b38c03bed4b4d0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -80,7 +80,7 @@ case _END_SEND: { _Py_UopsSymbol *value; - value = _Py_uop_sym_new_unknown(ctx); + value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; stack_pointer += -1; @@ -89,7 +89,7 @@ case _UNARY_NEGATIVE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -97,7 +97,7 @@ case _UNARY_NOT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -105,7 +105,7 @@ case _TO_BOOL: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -117,7 +117,7 @@ case _TO_BOOL_INT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -125,7 +125,7 @@ case _TO_BOOL_LIST: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -133,7 +133,7 @@ case _TO_BOOL_NONE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -141,7 +141,7 @@ case _TO_BOOL_STR: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -149,7 +149,7 @@ case _TO_BOOL_ALWAYS_TRUE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -157,7 +157,7 @@ case _UNARY_INVERT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -390,7 +390,7 @@ case _BINARY_OP_ADD_UNICODE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -399,7 +399,7 @@ case _BINARY_SUBSCR: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -408,7 +408,7 @@ case _BINARY_SLICE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -422,7 +422,7 @@ case _BINARY_SUBSCR_LIST_INT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -431,7 +431,7 @@ case _BINARY_SUBSCR_STR_INT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -440,7 +440,7 @@ case _BINARY_SUBSCR_TUPLE_INT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -449,7 +449,7 @@ case _BINARY_SUBSCR_DICT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -490,7 +490,7 @@ case _CALL_INTRINSIC_1: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -498,7 +498,7 @@ case _CALL_INTRINSIC_2: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -525,7 +525,7 @@ case _GET_AITER: { _Py_UopsSymbol *iter; - iter = _Py_uop_sym_new_unknown(ctx); + iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -533,7 +533,7 @@ case _GET_ANEXT: { _Py_UopsSymbol *awaitable; - awaitable = _Py_uop_sym_new_unknown(ctx); + awaitable = sym_new_unknown(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; stack_pointer += 1; @@ -542,7 +542,7 @@ case _GET_AWAITABLE: { _Py_UopsSymbol *iter; - iter = _Py_uop_sym_new_unknown(ctx); + iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -561,7 +561,7 @@ case _LOAD_ASSERTION_ERROR: { _Py_UopsSymbol *value; - value = _Py_uop_sym_new_unknown(ctx); + value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -570,7 +570,7 @@ case _LOAD_BUILD_CLASS: { _Py_UopsSymbol *bc; - bc = _Py_uop_sym_new_unknown(ctx); + bc = sym_new_unknown(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; stack_pointer += 1; @@ -604,7 +604,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = _Py_uop_sym_new_unknown(ctx); + values[_i] = sym_new_unknown(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -615,7 +615,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = _Py_uop_sym_new_unknown(ctx); + values[_i] = sym_new_unknown(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -626,7 +626,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = _Py_uop_sym_new_unknown(ctx); + values[_i] = sym_new_unknown(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -669,7 +669,7 @@ case _LOAD_LOCALS: { _Py_UopsSymbol *locals; - locals = _Py_uop_sym_new_unknown(ctx); + locals = sym_new_unknown(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; stack_pointer += 1; @@ -678,7 +678,7 @@ case _LOAD_FROM_DICT_OR_GLOBALS: { _Py_UopsSymbol *v; - v = _Py_uop_sym_new_unknown(ctx); + v = sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; break; @@ -686,7 +686,7 @@ case _LOAD_NAME: { _Py_UopsSymbol *v; - v = _Py_uop_sym_new_unknown(ctx); + v = sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; stack_pointer += 1; @@ -696,9 +696,9 @@ case _LOAD_GLOBAL: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - null = _Py_uop_sym_new_null(ctx); + null = sym_new_null(ctx); if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -717,9 +717,9 @@ case _LOAD_GLOBAL_MODULE: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - null = _Py_uop_sym_new_null(ctx); + null = sym_new_null(ctx); if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -730,9 +730,9 @@ case _LOAD_GLOBAL_BUILTINS: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; - null = _Py_uop_sym_new_null(ctx); + null = sym_new_null(ctx); if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -754,7 +754,7 @@ case _LOAD_FROM_DICT_OR_DEREF: { _Py_UopsSymbol *value; - value = _Py_uop_sym_new_unknown(ctx); + value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; @@ -762,7 +762,7 @@ case _LOAD_DEREF: { _Py_UopsSymbol *value; - value = _Py_uop_sym_new_unknown(ctx); + value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -780,7 +780,7 @@ case _BUILD_STRING: { _Py_UopsSymbol *str; - str = _Py_uop_sym_new_unknown(ctx); + str = sym_new_unknown(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -789,7 +789,7 @@ case _BUILD_TUPLE: { _Py_UopsSymbol *tup; - tup = _Py_uop_sym_new_unknown(ctx); + tup = sym_new_unknown(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -798,7 +798,7 @@ case _BUILD_LIST: { _Py_UopsSymbol *list; - list = _Py_uop_sym_new_unknown(ctx); + list = sym_new_unknown(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -817,7 +817,7 @@ case _BUILD_SET: { _Py_UopsSymbol *set; - set = _Py_uop_sym_new_unknown(ctx); + set = sym_new_unknown(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -826,7 +826,7 @@ case _BUILD_MAP: { _Py_UopsSymbol *map; - map = _Py_uop_sym_new_unknown(ctx); + map = sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -839,7 +839,7 @@ case _BUILD_CONST_KEY_MAP: { _Py_UopsSymbol *map; - map = _Py_uop_sym_new_unknown(ctx); + map = sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -865,7 +865,7 @@ case _LOAD_SUPER_ATTR_ATTR: { _Py_UopsSymbol *attr; - attr = _Py_uop_sym_new_unknown(ctx); + attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer += -2; @@ -875,9 +875,9 @@ case _LOAD_SUPER_ATTR_METHOD: { _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null; - attr = _Py_uop_sym_new_unknown(ctx); + attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; - self_or_null = _Py_uop_sym_new_unknown(ctx); + self_or_null = sym_new_unknown(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -888,9 +888,9 @@ case _LOAD_ATTR: { _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null = NULL; - attr = _Py_uop_sym_new_unknown(ctx); + attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; - self_or_null = _Py_uop_sym_new_unknown(ctx); + self_or_null = sym_new_unknown(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -1048,7 +1048,7 @@ case _COMPARE_OP: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1057,7 +1057,7 @@ case _COMPARE_OP_FLOAT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1066,7 +1066,7 @@ case _COMPARE_OP_INT: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1075,7 +1075,7 @@ case _COMPARE_OP_STR: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1084,7 +1084,7 @@ case _IS_OP: { _Py_UopsSymbol *b; - b = _Py_uop_sym_new_unknown(ctx); + b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1093,7 +1093,7 @@ case _CONTAINS_OP: { _Py_UopsSymbol *b; - b = _Py_uop_sym_new_unknown(ctx); + b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1103,9 +1103,9 @@ case _CHECK_EG_MATCH: { _Py_UopsSymbol *rest; _Py_UopsSymbol *match; - rest = _Py_uop_sym_new_unknown(ctx); + rest = sym_new_unknown(ctx); if (rest == NULL) goto out_of_space; - match = _Py_uop_sym_new_unknown(ctx); + match = sym_new_unknown(ctx); if (match == NULL) goto out_of_space; stack_pointer[-2] = rest; stack_pointer[-1] = match; @@ -1114,7 +1114,7 @@ case _CHECK_EXC_MATCH: { _Py_UopsSymbol *b; - b = _Py_uop_sym_new_unknown(ctx); + b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1126,7 +1126,7 @@ case _IS_NONE: { _Py_UopsSymbol *b; - b = _Py_uop_sym_new_unknown(ctx); + b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1134,7 +1134,7 @@ case _GET_LEN: { _Py_UopsSymbol *len_o; - len_o = _Py_uop_sym_new_unknown(ctx); + len_o = sym_new_unknown(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; stack_pointer += 1; @@ -1143,7 +1143,7 @@ case _MATCH_CLASS: { _Py_UopsSymbol *attrs; - attrs = _Py_uop_sym_new_unknown(ctx); + attrs = sym_new_unknown(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1152,7 +1152,7 @@ case _MATCH_MAPPING: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1161,7 +1161,7 @@ case _MATCH_SEQUENCE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1170,7 +1170,7 @@ case _MATCH_KEYS: { _Py_UopsSymbol *values_or_none; - values_or_none = _Py_uop_sym_new_unknown(ctx); + values_or_none = sym_new_unknown(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1179,7 +1179,7 @@ case _GET_ITER: { _Py_UopsSymbol *iter; - iter = _Py_uop_sym_new_unknown(ctx); + iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1187,7 +1187,7 @@ case _GET_YIELD_FROM_ITER: { _Py_UopsSymbol *iter; - iter = _Py_uop_sym_new_unknown(ctx); + iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1197,7 +1197,7 @@ case _FOR_ITER_TIER_TWO: { _Py_UopsSymbol *next; - next = _Py_uop_sym_new_unknown(ctx); + next = sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1218,7 +1218,7 @@ case _ITER_NEXT_LIST: { _Py_UopsSymbol *next; - next = _Py_uop_sym_new_unknown(ctx); + next = sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1237,7 +1237,7 @@ case _ITER_NEXT_TUPLE: { _Py_UopsSymbol *next; - next = _Py_uop_sym_new_unknown(ctx); + next = sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1270,9 +1270,9 @@ case _BEFORE_ASYNC_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = _Py_uop_sym_new_unknown(ctx); + exit = sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1283,9 +1283,9 @@ case _BEFORE_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = _Py_uop_sym_new_unknown(ctx); + exit = sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1295,7 +1295,7 @@ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1305,9 +1305,9 @@ case _PUSH_EXC_INFO: { _Py_UopsSymbol *prev_exc; _Py_UopsSymbol *new_exc; - prev_exc = _Py_uop_sym_new_unknown(ctx); + prev_exc = sym_new_unknown(ctx); if (prev_exc == NULL) goto out_of_space; - new_exc = _Py_uop_sym_new_unknown(ctx); + new_exc = sym_new_unknown(ctx); if (new_exc == NULL) goto out_of_space; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -1355,7 +1355,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _Py_UopsSymbol *attr; - attr = _Py_uop_sym_new_unknown(ctx); + attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1363,7 +1363,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _Py_UopsSymbol *attr; - attr = _Py_uop_sym_new_unknown(ctx); + attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1488,7 +1488,7 @@ case _CALL_TYPE_1: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1497,7 +1497,7 @@ case _CALL_STR_1: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1506,7 +1506,7 @@ case _CALL_TUPLE_1: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1522,7 +1522,7 @@ case _CALL_BUILTIN_CLASS: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1531,7 +1531,7 @@ case _CALL_BUILTIN_O: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1540,7 +1540,7 @@ case _CALL_BUILTIN_FAST: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1549,7 +1549,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1558,7 +1558,7 @@ case _CALL_LEN: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1567,7 +1567,7 @@ case _CALL_ISINSTANCE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1576,7 +1576,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1585,7 +1585,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1594,7 +1594,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1603,7 +1603,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1620,7 +1620,7 @@ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; - func = _Py_uop_sym_new_unknown(ctx); + func = sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; break; @@ -1628,7 +1628,7 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UopsSymbol *func; - func = _Py_uop_sym_new_unknown(ctx); + func = sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; stack_pointer += -1; @@ -1637,7 +1637,7 @@ case _BUILD_SLICE: { _Py_UopsSymbol *slice; - slice = _Py_uop_sym_new_unknown(ctx); + slice = sym_new_unknown(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1646,7 +1646,7 @@ case _CONVERT_VALUE: { _Py_UopsSymbol *result; - result = _Py_uop_sym_new_unknown(ctx); + result = sym_new_unknown(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; break; @@ -1654,7 +1654,7 @@ case _FORMAT_SIMPLE: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -1662,7 +1662,7 @@ case _FORMAT_WITH_SPEC: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1682,7 +1682,7 @@ case _BINARY_OP: { _Py_UopsSymbol *res; - res = _Py_uop_sym_new_unknown(ctx); + res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index e933fee8f4242b..d3ce4c8a25f5b8 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -28,7 +28,7 @@ from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import StackOffset, Stack, SizeMismatch, UNUSED +from stack import Stack, SizeMismatch, UNUSED DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" DEFAULT_ABSTRACT_INPUT = ROOT / "Python/optimizer_bytecodes.c" @@ -87,14 +87,14 @@ def emit_default(out: CWriter, uop: Uop) -> None: if var.name != "unused" and not var.peek: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = _Py_uop_sym_new_unknown(ctx);\n") + out.emit(f"{var.name}[_i] = sym_new_unknown(ctx);\n") out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") out.emit("}\n") elif var.name == "null": - out.emit(f"{var.name} = _Py_uop_sym_new_null(ctx);\n") + out.emit(f"{var.name} = sym_new_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") else: - out.emit(f"{var.name} = _Py_uop_sym_new_unknown(ctx);\n") + out.emit(f"{var.name} = sym_new_unknown(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") From fb2e17b642fc3089e4f98e4bf6b09dd362e6b27d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Feb 2024 10:42:28 +0100 Subject: [PATCH 52/82] gh-115937: Remove implementation details from inspect.signature() docs (#116086) Co-authored-by: Carol Willing Co-authored-by: Gregory P. Smith Co-authored-by: Jelle Zijlstra --- Doc/library/inspect.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8a74cadb98a0db..ed8d705da3b0b5 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -665,9 +665,6 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. - If the passed object has a ``__signature__`` attribute, this function - returns it without further computations. - For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using @@ -702,6 +699,13 @@ function. Python. For example, in CPython, some built-in functions defined in C provide no metadata about their arguments. + .. impl-detail:: + + If the passed object has a :attr:`!__signature__` attribute, + we may use it to create the signature. + The exact semantics are an implementation detail and are subject to + unannounced changes. Consult the source code for current semantics. + .. class:: Signature(parameters=None, *, return_annotation=Signature.empty) From 186fa9387669bcba6d3974a99c012c2b2c6fb4ce Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 29 Feb 2024 11:40:18 +0100 Subject: [PATCH 53/82] gh-116103: Prevent error in WindowsLoadTracker.__del__ on permission error (GH-116105) gh-116103: Prevent error in WindowsLoadTracker.__del__ if there was a permission error --- Lib/test/libregrtest/win_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/libregrtest/win_utils.py b/Lib/test/libregrtest/win_utils.py index 5736cdfd3c7292..b51fde0af57d1f 100644 --- a/Lib/test/libregrtest/win_utils.py +++ b/Lib/test/libregrtest/win_utils.py @@ -24,6 +24,10 @@ class WindowsLoadTracker(): """ def __init__(self): + # make __del__ not fail if pre-flight test fails + self._running = None + self._stopped = None + # Pre-flight test for access to the performance data; # `PermissionError` will be raised if not allowed winreg.QueryInfoKey(winreg.HKEY_PERFORMANCE_DATA) From bea2795be2c08dde3830f987830414f3f12bc1eb Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 29 Feb 2024 03:09:09 -0800 Subject: [PATCH 54/82] gh-115881: Document feature_version limitations (#115980) --- Doc/library/ast.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index d629393f023c8a..5ed6eb63c0fb62 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2183,12 +2183,15 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 7)``; the highest is ``sys.version_info[0:2]``. + Setting ``feature_version`` to a tuple ``(major, minor)`` will result in + a "best-effort" attempt to parse using that Python version's grammar. + For example, setting ``feature_version=(3, 9)`` will attempt to disallow + parsing of :keyword:`match` statements. + Currently ``major`` must equal to ``3``. The lowest supported version is + ``(3, 7)`` (and this may increase in future Python versions); + the highest is ``sys.version_info[0:2]``. "Best-effort" attempt means there + is no guarantee that the parse (or success of the parse) is the same as + when run on the Python version corresponding to ``feature_version``. If source contains a null character ('\0'), :exc:`ValueError` is raised. From 6a86030bc2519b4a6b055e0b47b9870c86db8588 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 29 Feb 2024 16:59:24 +0200 Subject: [PATCH 55/82] gh-116100: Add `test` arg to `ast.If` and `op` arg to `ast.BoolOp` calls (#116101) --- Lib/test/test_compile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 0126780982059a..d3e69bfedccd07 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -527,11 +527,11 @@ def test_compile_ast(self): self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') def test_compile_invalid_typealias(self): From 45d8871dc4da33fcef92991031707c5bf88a40cf Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Thu, 29 Feb 2024 09:38:04 -0600 Subject: [PATCH 56/82] gh-112844: Add SBOM for external dependencies (#115789) --- .github/CODEOWNERS | 1 + Misc/externals.spdx.json | 174 +++++++++++++++++++++++++++++++++++ Tools/build/generate_sbom.py | 108 ++++++++++++++++++---- 3 files changed, 266 insertions(+), 17 deletions(-) create mode 100644 Misc/externals.spdx.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1d0bce111320ff..e8eed400d961fc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -247,6 +247,7 @@ Lib/test/test_interpreters/ @ericsnowcurrently /Tools/wasm/ @brettcannon # SBOM +/Misc/externals.spdx.json @sethmlarson /Misc/sbom.spdx.json @sethmlarson /Tools/build/generate_sbom.py @sethmlarson diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json new file mode 100644 index 00000000000000..2acfccbb004d6b --- /dev/null +++ b/Misc/externals.spdx.json @@ -0,0 +1,174 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-bzip2", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab8d1b0cc087c20d4c32c0e4fcf7d0c733a95da12cedc6d63b3f0a9af07427e2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/bzip2-1.0.8.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bzip:bzip2:1.0.8:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "bzip2", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.8" + }, + { + "SPDXID": "SPDXRef-PACKAGE-libffi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9d802681adfea27d84cae0487a785fb9caa925bdad44c401b364c59ab2b8edda" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.4.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:libffi_project:libffi:3.4.4:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "libffi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-openssl", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "openssl", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.0.13" + }, + { + "SPDXID": "SPDXRef-PACKAGE-sqlite", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "6f0364a27375435a34137b138ca4fedef8d23eec6493ca1dfff33bfc0c34fda4" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.1.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.1.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "sqlite", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.45.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "1d3f2015e49e269cf681373d433cd54d88d5ef7443fe87f5f50f5fcfe9003e73" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tcl-core", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tk", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "6056203b8a6aaf6ea89d90a7b55dc7f407e55c093f731a98fd830a712a3c81d3" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tk", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-xz", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:xz_project:xz:5.2.5:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "xz", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.2.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "zlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.3.1" + } + ], + "spdxVersion": "SPDX-2.3" +} \ No newline at end of file diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 201c81c4d14d79..6aa4946ee227e7 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -7,9 +7,8 @@ import pathlib import subprocess import sys +import urllib.request import typing -import zipfile -from urllib.request import urlopen CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent @@ -125,30 +124,41 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) -def main() -> None: - sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" - sbom_data = json.loads(sbom_path.read_bytes()) +def get_externals() -> list[str]: + """ + Parses 'PCbuild/get_externals.bat' for external libraries. + Returns a list of (git tag, name, version) tuples. + """ + get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat" + externals = re.findall( + r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s", + get_externals_bat_path.read_text() + ) + return externals - # We regenerate all of this information. Package information - # should be preserved though since that is edited by humans. - sbom_data["files"] = [] - sbom_data["relationships"] = [] - # Ensure all packages in this tool are represented also in the SBOM file. - actual_names = {package["name"] for package in sbom_data["packages"]} - expected_names = set(PACKAGE_TO_FILES) - error_if( - actual_names != expected_names, - f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", - ) +def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: + """Make a bunch of assertions about the SBOM package data to ensure it's consistent.""" - # Make a bunch of assertions about the SBOM data to ensure it's consistent. for package in sbom_data["packages"]: # Properties and ID must be properly formed. error_if( "name" not in package, "Package is missing the 'name' field" ) + + # Verify that the checksum matches the expected value + # and that the download URL is valid. + if "checksums" not in package or "CI" in os.environ: + download_location = package["downloadLocation"] + resp = urllib.request.urlopen(download_location) + error_if(resp.status != 200, f"Couldn't access URL: {download_location}'") + + package["checksums"] = [{ + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(resp.read()).hexdigest() + }] + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) error_if( bool(missing_required_keys), @@ -180,6 +190,26 @@ def main() -> None: f"License identifier must be 'NOASSERTION'" ) + +def create_source_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + check_sbom_packages(sbom_data) + # We call 'sorted()' here a lot to avoid filesystem scan order issues. for name, files in sorted(PACKAGE_TO_FILES.items()): package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") @@ -224,5 +254,49 @@ def main() -> None: sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) +def create_externals_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + externals = get_externals() + externals_name_to_version = {} + externals_name_to_git_tag = {} + for git_tag in externals: + name, _, version = git_tag.rpartition("-") + externals_name_to_version[name] = version + externals_name_to_git_tag[name] = git_tag + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(externals_name_to_version) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + # Set the versionInfo and downloadLocation fields for all packages. + for package in sbom_data["packages"]: + package["versionInfo"] = externals_name_to_version[package["name"]] + download_location = ( + f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz" + ) + download_location_changed = download_location != package["downloadLocation"] + package["downloadLocation"] = download_location + + # If the download URL has changed we want one to get recalulated. + if download_location_changed: + package.pop("checksums", None) + + check_sbom_packages(sbom_data) + + # Update the SBOM on disk + sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) + + +def main() -> None: + create_source_sbom() + create_externals_sbom() + + if __name__ == "__main__": main() From f0df35eeca2ccdfd58cfb9801f06ffa23537270b Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 29 Feb 2024 08:11:28 -0800 Subject: [PATCH 57/82] GH-115802: JIT "small" code for Windows (GH-115964) --- Include/cpython/optimizer.h | 3 -- Include/internal/pycore_ceval.h | 30 +++++++++------- Include/internal/pycore_dict.h | 10 +++--- Include/internal/pycore_floatobject.h | 2 +- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_genobject.h | 4 +-- Include/internal/pycore_intrinsics.h | 4 +-- Include/internal/pycore_list.h | 6 ++-- Include/internal/pycore_long.h | 6 ++-- Include/internal/pycore_object.h | 4 +-- Include/internal/pycore_optimizer.h | 2 ++ Include/internal/pycore_pyerrors.h | 8 ++--- Include/internal/pycore_sliceobject.h | 2 +- Include/internal/pycore_tuple.h | 2 +- Include/internal/pycore_typeobject.h | 2 +- Include/internal/pycore_unicodeobject.h | 4 +-- Python/bytecodes.c | 8 ++--- Python/ceval.c | 6 ++++ Python/ceval_macros.h | 7 ---- Python/executor_cases.c.h | 8 ++--- Python/generated_cases.c.h | 8 ++--- Python/jit.c | 26 ++++++++------ Tools/jit/_schema.py | 5 ++- Tools/jit/_stencils.py | 9 ++++- Tools/jit/_targets.py | 46 +++++++++++++++++-------- Tools/jit/template.c | 5 +-- 26 files changed, 126 insertions(+), 93 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 8fc9fb62aebdb4..6d7b8bc3c1433a 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -92,9 +92,6 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); -int -_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); - void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorClear(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index bf77526cf75cc1..6eab2ba1daedf8 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -181,22 +181,26 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ -extern int _Py_HandlePending(PyThreadState *tstate); +PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); extern PyObject * _PyEval_GetFrameLocals(void); -extern const binaryfunc _PyEval_BinaryOps[]; -int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); -void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); -void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); -void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); -PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); -PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); -int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); -void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); +typedef PyObject *(*conversion_func)(PyObject *); + +PyAPI_DATA(const binaryfunc) _PyEval_BinaryOps[]; +PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[]; + +PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); +PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); +PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); +PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); +PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); /* Bits that can be set in PyThreadState.eval_breaker */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index d1a0010b9a81dd..cd171a4384db5d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -52,7 +52,7 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); extern void _PyDict_DebugMallocStats(FILE *out); @@ -100,10 +100,10 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); -extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); /* Consumes references to key and value */ -extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( @@ -247,8 +247,8 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); -extern PyObject *_PyDict_FromItems( +PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 3767df5506d43f..f984df695696c3 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -34,7 +34,7 @@ struct _Py_float_runtime_state { -void _PyFloat_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op); extern void _PyFloat_DebugMallocStats(FILE* out); diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 3f3da8a44b77e4..dad6a89af77dec 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -29,7 +29,7 @@ struct _py_func_state { extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); -extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); extern PyObject *_Py_set_function_type_params( diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index b2aa017598409f..9463c822ad8669 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -10,7 +10,7 @@ extern "C" { #include "pycore_freelist.h" -extern PyObject *_PyGen_yf(PyGenObject *); +PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension @@ -19,7 +19,7 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o); +PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 3a8dd95cff8e5d..8fa88ea3f74caa 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -44,7 +44,7 @@ typedef struct { const char *name; } intrinsic_func2_info; -extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; -extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; +PyAPI_DATA(const intrinsic_func1_info) _PyIntrinsics_UnaryFunctions[]; +PyAPI_DATA(const intrinsic_func2_info) _PyIntrinsics_BinaryFunctions[]; #endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 50dc13c4da4487..2a82912e41d557 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -10,12 +10,12 @@ extern "C" { #include "pycore_freelist.h" // _PyFreeListState -extern PyObject* _PyList_Extend(PyListObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) -extern int +PyAPI_FUNC(int) _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); // In free-threaded build: self should be locked by the caller, if it should be thread-safe. @@ -54,7 +54,7 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } _PyListIterObject; -extern PyObject *_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); +PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); #ifdef __cplusplus } diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ec27df9e416c58..f04f66d053bab9 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -121,9 +121,9 @@ PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); -extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 34a83ea228e8b1..9809f5f2e0271a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -73,7 +73,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); .ob_size = size \ } -extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, const char *message); @@ -684,7 +684,7 @@ PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); extern int _PyObject_IsAbstract(PyObject *); -extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); extern PyObject* _PyObject_NextNotImplemented(PyObject *); // Pickle support. diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 614850468ec1d3..4894f613b5fb91 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -111,6 +111,8 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 0f16fb894d17e1..910335fd2cf33b 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -95,7 +95,7 @@ extern void _PyErr_Fetch( extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); -extern int _PyErr_ExceptionMatches( +PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); @@ -114,18 +114,18 @@ extern void _PyErr_SetObject( extern void _PyErr_ChainStackItem(void); -extern void _PyErr_Clear(PyThreadState *tstate); +PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate); extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -extern void _PyErr_SetString( +PyAPI_FUNC(void) _PyErr_SetString( PyThreadState *tstate, PyObject *exception, const char *string); -extern PyObject* _PyErr_Format( +PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, const char *format, diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 89086f67683a2f..ba8b1f1cb27dee 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,7 +11,7 @@ extern "C" { /* runtime lifecycle */ -extern PyObject * +PyAPI_FUNC(PyObject *) _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); #ifdef __cplusplus diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 4605f355ccbc38..14a9e42c3a324c 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,7 +21,7 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); -extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 9134ab45cd0039..c214111fed6f97 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -147,7 +147,7 @@ extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); extern PyTypeObject _PyBufferWrapper_Type; -extern PyObject* _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, +PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 7ee540154b23d8..fea5ceea0954f4 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -31,7 +31,7 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); -extern void _PyUnicode_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op); extern Py_ssize_t _PyUnicode_InternedSize(void); // Get a copy of a Unicode string. @@ -202,7 +202,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ -extern PyObject* _PyUnicode_JoinArray( +PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( PyObject *separator, PyObject *const *items, Py_ssize_t seqlen diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 565379afc4b5a7..1d515098f6c7e9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2755,7 +2755,7 @@ dummy_func( GOTO_ERROR(error); } DECREF_INPUTS(); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2790,7 +2790,7 @@ dummy_func( GOTO_ERROR(error); } DECREF_INPUTS(); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -3822,9 +3822,9 @@ dummy_func( } inst(CONVERT_VALUE, (value -- result)) { - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; + conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); ERROR_IF(result == NULL, error); diff --git a/Python/ceval.c b/Python/ceval.c index 41e9310938d826..34f286e4e17eb8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -337,6 +337,12 @@ const binaryfunc _PyEval_BinaryOps[] = { [NB_INPLACE_XOR] = PyNumber_InPlaceXor, }; +const conversion_func _PyEval_ConversionFuncs[4] = { + [FVC_STR] = PyObject_Str, + [FVC_REPR] = PyObject_Repr, + [FVC_ASCII] = PyObject_ASCII +}; + // PEP 634: Structural Pattern Matching diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 6ad41950ea3b7e..9674a7824a4690 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -352,13 +352,6 @@ do { \ } \ } while (0); -typedef PyObject *(*convertion_func_ptr)(PyObject *); - -static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { - [FVC_STR] = PyObject_Str, - [FVC_REPR] = PyObject_Repr, - [FVC_ASCII] = PyObject_ASCII -}; // GH-89279: Force inlining by using a macro. #if defined(_MSC_VER) && SIZEOF_INT == 4 diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 20fab8f4c61eb5..9ec1be9076a5a0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2548,7 +2548,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2591,7 +2591,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -3570,9 +3570,9 @@ PyObject *result; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; + conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error_tier_two; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bb26dac0e2be20..3312078e9a2d4d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -40,7 +40,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -86,7 +86,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2140,9 +2140,9 @@ PyObject *value; PyObject *result; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; + conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; diff --git a/Python/jit.c b/Python/jit.c index ac2c60ed925a26..9f9e123ab91fef 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -203,13 +203,14 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) *loc32 = (uint32_t)value; continue; case HoleKind_ARM64_RELOC_UNSIGNED: - case HoleKind_IMAGE_REL_AMD64_ADDR64: case HoleKind_R_AARCH64_ABS64: case HoleKind_X86_64_RELOC_UNSIGNED: case HoleKind_R_X86_64_64: // 64-bit absolute address. *loc64 = value; continue; + case HoleKind_IMAGE_REL_AMD64_REL32: + case HoleKind_IMAGE_REL_I386_REL32: case HoleKind_R_X86_64_GOTPCRELX: case HoleKind_R_X86_64_REX_GOTPCRELX: case HoleKind_X86_64_RELOC_GOT: @@ -249,7 +250,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); assert((int64_t)value < (1LL << 31)); - loc32[0] = (uint32_t)value; + *loc32 = (uint32_t)value; continue; case HoleKind_R_AARCH64_CALL26: case HoleKind_R_AARCH64_JUMP26: @@ -307,23 +308,23 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) next_hole->addend == hole->addend && next_hole->value == hole->value) { - unsigned char rd = get_bits(loc32[0], 0, 5); + unsigned char reg = get_bits(loc32[0], 0, 5); assert(IS_AARCH64_LDR_OR_STR(loc32[1])); - unsigned char rt = get_bits(loc32[1], 0, 5); - unsigned char rn = get_bits(loc32[1], 5, 5); - assert(rd == rn && rn == rt); + // There should be only one register involved: + assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. + assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. uint64_t relaxed = *(uint64_t *)value; if (relaxed < (1UL << 16)) { // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd; + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; loc32[1] = 0xD503201F; i++; continue; } if (relaxed < (1ULL << 32)) { // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd; - loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | rd; + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; i++; continue; } @@ -332,13 +333,15 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) (int64_t)relaxed >= -(1L << 19) && (int64_t)relaxed < (1L << 19)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr x0, XXX; nop - loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | rd; + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; loc32[1] = 0xD503201F; i++; continue; } } + // Fall through... + case HoleKind_ARM64_RELOC_PAGE21: // Number of pages between this page and the value's page: value = (value >> 12) - ((uint64_t)location >> 12); // Check that we're not out of range of 21 signed bits: @@ -350,6 +353,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) set_bits(loc32, 5, value, 2, 19); continue; case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: + case HoleKind_ARM64_RELOC_PAGEOFF12: case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: // 12-bit low part of an absolute address. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGE21 (above). diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 975ca650a13c1a..14e5fc2aae80ef 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -4,9 +4,12 @@ HoleKind: typing.TypeAlias = typing.Literal[ "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_PAGE21", + "ARM64_RELOC_PAGEOFF12", "ARM64_RELOC_UNSIGNED", - "IMAGE_REL_AMD64_ADDR64", + "IMAGE_REL_AMD64_REL32", "IMAGE_REL_I386_DIR32", + "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", "R_AARCH64_ADR_GOT_PAGE", "R_AARCH64_CALL26", diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 71c678e04fbfd5..eddec731984c82 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -96,7 +96,7 @@ def emit_aarch64_trampoline(self, hole: Hole) -> None: instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF self.body[where] = instruction.to_bytes(4, sys.byteorder) self.disassembly += [ - f"{base + 4 * 0: x}: d2800008 mov x8, #0x0", + f"{base + 4 * 0:x}: d2800008 mov x8, #0x0", f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}", f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16", f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}", @@ -162,6 +162,13 @@ def process_relocations(self, *, alignment: int = 1) -> None: ): self.code.emit_aarch64_trampoline(hole) continue + elif ( + hole.kind in {"IMAGE_REL_AMD64_REL32"} + and hole.value is HoleValue.ZERO + ): + raise ValueError( + f"Add PyAPI_FUNC(...) or PyAPI_DATA(...) to declaration of {hole.symbol}!" + ) holes.append(hole) stencil.holes[:] = holes self.code.pad(alignment) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 06dc4e7acc6c91..07959b15b6c4b9 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -106,7 +106,7 @@ async def _compile( o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", - "-DPy_BUILD_CORE", + "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", @@ -118,12 +118,17 @@ async def _compile( f"-I{CPYTHON / 'Python'}", "-O3", "-c", + # This debug info isn't necessary, and bloats out the JIT'ed code. + # We *may* be able to re-enable this, process it, and JIT it for a + # nicer debugging experience... but that needs a lot more research: "-fno-asynchronous-unwind-tables", + # Don't call built-in functions that we can't find or patch: "-fno-builtin", - # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: - "-fno-jump-tables", + # Emit relaxable 64-bit calls/jumps, so we don't have to worry about + # about emitting in-range trampolines for out-of-range targets. + # We can probably remove this and emit trampolines in the future: "-fno-plt", - # Don't make calls to weird stack-smashing canaries: + # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-o", f"{o}", @@ -194,12 +199,21 @@ def _handle_section( offset = base + symbol["Value"] name = symbol["Name"] name = name.removeprefix(self.prefix) - group.symbols[name] = value, offset + if name not in group.symbols: + group.symbols[name] = value, offset for wrapped_relocation in section["Relocations"]: relocation = wrapped_relocation["Relocation"] hole = self._handle_relocation(base, relocation, stencil.body) stencil.holes.append(hole) + def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: + if name.startswith("__imp_"): + name = name.removeprefix("__imp_") + name = name.removeprefix(self.prefix) + return _stencils.HoleValue.GOT, name + name = name.removeprefix(self.prefix) + return _stencils.symbol_to_value(name) + def _handle_relocation( self, base: int, relocation: _schema.COFFRelocation, raw: bytes ) -> _stencils.Hole: @@ -207,21 +221,23 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind}, + "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base - s = s.removeprefix(self.prefix) - value, symbol = _stencils.symbol_to_value(s) - addend = int.from_bytes(raw[offset : offset + 8], "little") + value, symbol = self._unwrap_dllimport(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": { + "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + }, }: offset += base - s = s.removeprefix(self.prefix) - value, symbol = _stencils.symbol_to_value(s) - addend = int.from_bytes(raw[offset : offset + 4], "little") + value, symbol = self._unwrap_dllimport(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) case _: raise NotImplementedError(relocation) return _stencils.Hole(offset, kind, value, symbol, addend) @@ -423,12 +439,12 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: args = ["-mcmodel=large"] return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): - args = ["-mcmodel=large"] + args = ["-DPy_NO_ENABLE_SHARED"] return _COFF(host, args=args, prefix="_") if re.fullmatch(r"x86_64-apple-darwin.*", host): return _MachO(host, prefix="_") if re.fullmatch(r"x86_64-pc-windows-msvc", host): - args = ["-mcmodel=large"] + args = ["-fms-runtime-lib=dll"] return _COFF(host, args=args) if re.fullmatch(r"x86_64-.*-linux-gnu", host): return _ELF(host) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index d79c6efb8f6de4..8aaf4581de362d 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -9,6 +9,7 @@ #include "pycore_long.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" #include "pycore_range.h" #include "pycore_setobject.h" #include "pycore_sliceobject.h" @@ -58,11 +59,11 @@ do { \ } while (0) #define PATCH_VALUE(TYPE, NAME, ALIAS) \ - extern void ALIAS; \ + PyAPI_DATA(void) ALIAS; \ TYPE NAME = (TYPE)(uint64_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ - extern void ALIAS; \ + PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); From a81d9509ee60c405e08012aec5d97da5840b590b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 29 Feb 2024 11:30:18 -0600 Subject: [PATCH 58/82] Make the iter_except() recipe more compact. (gh-116132) Only one example is needed --- Doc/library/itertools.rst | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c26f6c89b4920a..072fe024227ad2 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -983,28 +983,10 @@ which incur interpreter overhead. """ Call a function repeatedly until an exception is raised. Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Priority queue iterator: - iter_except(functools.partial(heappop, h), IndexError) - - Non-blocking dictionary iterator: - iter_except(d.popitem, KeyError) - - Non-blocking deque iterator: - iter_except(d.popleft, IndexError) - - Non-blocking iterator over a producer Queue: - iter_except(q.get_nowait, Queue.Empty) - - Non-blocking set iterator: - iter_except(s.pop, KeyError) - """ + # iter_except(d.popitem, KeyError) --> non-blocking dictionary iterator try: if first is not None: - # For database APIs needing an initial call to db.first() yield first() while True: yield func() @@ -1012,7 +994,6 @@ which incur interpreter overhead. pass - The following recipes have a more mathematical flavor: .. testcode:: From 04d1000071b5eefd4b1dd69051aacad343da4b21 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Thu, 29 Feb 2024 19:37:31 +0100 Subject: [PATCH 59/82] gh-72463: Fix ctypes/test_loading.py so that test_find reports skipped (GH-18312) --- Lib/test/test_ctypes/test_loading.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 59d7f51935f3cd..b218e9e7720c79 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -59,11 +59,15 @@ def test_load_version(self): self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): + found = False for name in ("c", "m"): lib = find_library(name) if lib: + found = True cdll.LoadLibrary(lib) CDLL(lib) + if not found: + self.skipTest("Could not find c and m libraries") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') From 3b6f4cadf19e6a4edd2cbbbc96a0a4024b395648 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 29 Feb 2024 18:53:19 +0000 Subject: [PATCH 60/82] gh-115811: Update documentation to add some Logger attributes. (GH-116109) --- Doc/library/logging.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 39eb41ce1f1670..e38b7435498507 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -77,6 +77,27 @@ is the module's name in the Python package namespace. .. class:: Logger + .. attribute:: Logger.name + + This is the logger's name, and is the value that was passed to :func:`getLogger` + to obtain the logger. + + .. note:: This attribute should be treated as read-only. + + .. attribute:: Logger.level + + The threshold of this logger, as set by the :meth:`setLevel` method. + + .. note:: Do not set this attribute directly - always use :meth:`setLevel`, + which has checks for the level passed to it. + + .. attribute:: Logger.parent + + The parent logger of this logger. It may change based on later instantiation + of loggers which are higher up in the namespace hierarchy. + + .. note:: This value should be treated as read-only. + .. attribute:: Logger.propagate If this attribute evaluates to true, events logged to this logger will be @@ -108,6 +129,21 @@ is the module's name in the Python package namespace. scenario is to attach handlers only to the root logger, and to let propagation take care of the rest. + .. attribute:: Logger.handlers + + The list of handlers directly attached to this logger instance. + + .. note:: This attribute should be treated as read-only; it is normally changed via + the :meth:`addHandler` and :meth:`removeHandler` methods, which use locks to ensure + thread-safe operation. + + .. attribute:: Logger.disabled + + This attribute disables handling of any events. It is set to ``False`` in the + initializer, and only changed by logging configuration code. + + .. note:: This attribute should be treated as read-only. + .. method:: Logger.setLevel(level) Sets the threshold for this logger to *level*. Logging messages which are less From 0656509033948780e6703391daca773c779041f7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 29 Feb 2024 10:55:29 -0800 Subject: [PATCH 61/82] gh-116088: Insert bottom checks after all sym_set_...() calls (#116089) This changes the `sym_set_...()` functions to return a `bool` which is `false` when the symbol is `bottom` after the operation. All calls to such functions now check this result and go to `hit_bottom`, a special error label that prints a different message and then reports that it wasn't able to optimize the trace. No executor will be produced in this case. --- Include/internal/pycore_optimizer.h | 8 ++--- Lib/test/test_capi/test_opt.py | 24 +++++++++---- Python/optimizer_analysis.c | 9 +++++ Python/optimizer_bytecodes.c | 36 +++++++++++++++----- Python/optimizer_cases.c.h | 36 +++++++++++++++----- Python/optimizer_symbols.c | 36 ++++++++++++-------- Tools/cases_generator/optimizer_generator.py | 4 --- 7 files changed, 106 insertions(+), 47 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 4894f613b5fb91..d32e6c0174f680 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -91,10 +91,10 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_type( extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern void _Py_uop_sym_set_null(_Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern void _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index a43726f05a448d..d21765307f0f09 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -894,23 +894,33 @@ def testfunc(n): def test_type_inconsistency(self): ns = {} - exec(textwrap.dedent(""" + src = textwrap.dedent(""" def testfunc(n): for i in range(n): x = _test_global + _test_global - """), globals(), ns) + """) + exec(src, ns, ns) testfunc = ns['testfunc'] - # Must be a real global else it won't be optimized to _LOAD_CONST_INLINE - global _test_global - _test_global = 0 + ns['_test_global'] = 0 _, ex = self._run_with_optimizer(testfunc, 16) self.assertIsNone(ex) - _test_global = 1.2 + ns['_test_global'] = 1 _, ex = self._run_with_optimizer(testfunc, 16) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_BOTH_INT", uops) + self.assertNotIn("_GUARD_BOTH_INT", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) + # Try again, but between the runs, set the global to a float. + # This should result in no executor the second time. + ns = {} + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) + ns['_test_global'] = 3.14 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) if __name__ == "__main__": diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 2a7ef4ec919eeb..a326e2249bb4de 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -363,6 +363,15 @@ optimize_uops( DPRINTF(1, "Encountered error in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; + +hit_bottom: + // Attempted to push a "bottom" (contradition) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's + // simpler to just admit failure and not create the executor. + DPRINTF(1, "Hit bottom in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 928c22da16b999..aa19a50f09ff71 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -85,8 +85,12 @@ dummy_func(void) { sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); + if (!sym_set_type(left, &PyLong_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyLong_Type)) { + goto hit_bottom; + } } op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { @@ -94,8 +98,12 @@ dummy_func(void) { sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); + if (!sym_set_type(left, &PyFloat_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyFloat_Type)) { + goto hit_bottom; + } } op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { @@ -103,8 +111,12 @@ dummy_func(void) { sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(right, &PyUnicode_Type); + if (!sym_set_type(left, &PyUnicode_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyUnicode_Type)) { + goto hit_bottom; + } } op(_BINARY_OP_ADD_INT, (left, right -- res)) { @@ -365,14 +377,20 @@ dummy_func(void) { op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - sym_set_type(callable, &PyFunction_Type); + if (!sym_set_type(callable, &PyFunction_Type)) { + goto hit_bottom; + } (void)self_or_null; (void)func_version; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + if (!sym_set_null(null)) { + goto hit_bottom; + } + if (!sym_set_type(callable, &PyMethod_Type)) { + goto hit_bottom; + } } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b38c03bed4b4d0..b34c57376df88e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -172,8 +172,12 @@ sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); + if (!sym_set_type(left, &PyLong_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyLong_Type)) { + goto hit_bottom; + } break; } @@ -276,8 +280,12 @@ sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); + if (!sym_set_type(left, &PyFloat_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyFloat_Type)) { + goto hit_bottom; + } break; } @@ -383,8 +391,12 @@ sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyUnicode_Type); - sym_set_type(right, &PyUnicode_Type); + if (!sym_set_type(left, &PyUnicode_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyUnicode_Type)) { + goto hit_bottom; + } break; } @@ -1397,8 +1409,12 @@ _Py_UopsSymbol *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + if (!sym_set_null(null)) { + goto hit_bottom; + } + if (!sym_set_type(callable, &PyMethod_Type)) { + goto hit_bottom; + } break; } @@ -1425,7 +1441,9 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand; - sym_set_type(callable, &PyFunction_Type); + if (!sym_set_type(callable, &PyFunction_Type)) { + goto hit_bottom; + } (void)self_or_null; (void)func_version; break; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index a529cc2f5cf215..5c3ec2b5ed1a4c 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -113,60 +113,68 @@ _Py_uop_sym_get_const(_Py_UopsSymbol *sym) return sym->const_val; } -void +bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); if (sym->flags & IS_NULL) { sym_set_bottom(sym); - return; + return false; } if (sym->typ != NULL) { if (sym->typ != typ) { sym_set_bottom(sym); + return false; } - return; } - sym_set_flag(sym, NOT_NULL); - sym->typ = typ; + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + } + return true; } -void +bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) { assert(const_val != NULL); if (sym->flags & IS_NULL) { sym_set_bottom(sym); - return; + return false; } PyTypeObject *typ = Py_TYPE(const_val); if (sym->typ != NULL && sym->typ != typ) { sym_set_bottom(sym); - return; + return false; } if (sym->const_val != NULL) { if (sym->const_val != const_val) { // TODO: What if they're equal? sym_set_bottom(sym); + return false; } - return; } - sym_set_flag(sym, NOT_NULL); - sym->typ = typ; - sym->const_val = Py_NewRef(const_val); + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); + } + return true; } -void +bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym) { sym_set_flag(sym, IS_NULL); + return !_Py_uop_sym_is_bottom(sym); } -void +bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) { sym_set_flag(sym, NOT_NULL); + return !_Py_uop_sym_is_bottom(sym); } diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index d3ce4c8a25f5b8..fca42b51fbd689 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -4,16 +4,12 @@ """ import argparse -import os.path -import sys from analyzer import ( Analysis, Instruction, Uop, - Part, analyze_files, - Skip, StackItem, analysis_error, ) From dbe44f150cd161bd327ed662e527a4c93829fb3e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 29 Feb 2024 21:46:12 +0100 Subject: [PATCH 62/82] Docs: mark up NotImplemented using the :data: role throughout the docs (#116135) --- Doc/library/abc.rst | 4 ++-- Doc/library/constants.rst | 14 +++++++------- Doc/library/exceptions.rst | 4 ++-- Doc/library/importlib.rst | 2 +- Doc/library/numbers.rst | 2 +- Doc/library/pickle.rst | 6 +++--- Doc/library/stdtypes.rst | 6 +++--- Doc/library/unittest.mock.rst | 8 ++++---- Doc/reference/datamodel.rst | 18 +++++++++--------- Doc/reference/expressions.rst | 2 +- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.10.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- Doc/whatsnew/3.9.rst | 2 +- Misc/NEWS.d/3.10.0a6.rst | 2 +- Misc/NEWS.d/3.11.0a1.rst | 2 +- 16 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index c073ea955abaa4..10e2cba50e49b0 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -101,11 +101,11 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: subclass of the ABC. (This class method is called from the :meth:`~class.__subclasscheck__` method of the ABC.) - This method should return ``True``, ``False`` or ``NotImplemented``. If + This method should return ``True``, ``False`` or :data:`NotImplemented`. If it returns ``True``, the *subclass* is considered a subclass of this ABC. If it returns ``False``, the *subclass* is not considered a subclass of this ABC, even if it would normally be one. If it returns - ``NotImplemented``, the subclass check is continued with the usual + :data:`!NotImplemented`, the subclass check is continued with the usual mechanism. .. XXX explain the "usual mechanism" diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 401dc9a320c5e0..93a7244f87de6b 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -33,27 +33,27 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. .. note:: - When a binary (or in-place) method returns ``NotImplemented`` the + When a binary (or in-place) method returns :data:`!NotImplemented` the interpreter will try the reflected operation on the other type (or some other fallback, depending on the operator). If all attempts return - ``NotImplemented``, the interpreter will raise an appropriate exception. - Incorrectly returning ``NotImplemented`` will result in a misleading - error message or the ``NotImplemented`` value being returned to Python code. + :data:`!NotImplemented`, the interpreter will raise an appropriate exception. + Incorrectly returning :data:`!NotImplemented` will result in a misleading + error message or the :data:`!NotImplemented` value being returned to Python code. See :ref:`implementing-the-arithmetic-operations` for examples. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`!NotImplemented` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While + Evaluating :data:`!NotImplemented` in a boolean context is deprecated. While it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. It will raise a :exc:`TypeError` in a future version of Python. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 88417b40e4aa7f..7879fb015bddfa 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -335,9 +335,9 @@ The following exceptions are the exceptions that are usually raised. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`NotImplemented` are not interchangeable, even though they have similar names and purposes. See - :data:`NotImplemented` for details on when to use it. + :data:`!NotImplemented` for details on when to use it. .. exception:: OSError([arg]) OSError(errno, strerror[, filename[, winerror[, filename2]]]) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2402bc5cd3ee2c..d92bb2f8e5cf83 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -265,7 +265,7 @@ ABC hierarchy:: when invalidating the caches of all finders on :data:`sys.meta_path`. .. versionchanged:: 3.4 - Returns ``None`` when called instead of ``NotImplemented``. + Returns ``None`` when called instead of :data:`NotImplemented`. .. class:: PathEntryFinder diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 17d1a275f04c9b..306bdd94aaca13 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -166,7 +166,7 @@ Complex``. I'll consider ``a + b``: 2. If ``A`` falls back to the boilerplate code, and it were to return a value from :meth:`~object.__add__`, we'd miss the possibility that ``B`` defines a more intelligent :meth:`~object.__radd__`, so the - boilerplate should return :const:`NotImplemented` from + boilerplate should return :data:`NotImplemented` from :meth:`!__add__`. (Or ``A`` may not implement :meth:`!__add__` at all.) 3. Then ``B``'s :meth:`~object.__radd__` gets a chance. If it accepts diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index cb517681fa81b9..223c27237e4d34 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -377,7 +377,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It should conform to the same interface as a :meth:`~object.__reduce__` method, and - can optionally return ``NotImplemented`` to fallback on + can optionally return :data:`NotImplemented` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. For a detailed example, see :ref:`reducer_override`. @@ -503,7 +503,7 @@ What can be pickled and unpickled? The following types can be pickled: * built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and - ``NotImplemented``); + :data:`NotImplemented`); * integers, floating-point numbers, complex numbers; @@ -905,7 +905,7 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return -``NotImplemented`` to fallback to the traditional behavior. +:data:`NotImplemented` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and :meth:`~Pickler.reducer_override` are defined, then diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1a4c12590c1018..c81626a194e640 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5450,10 +5450,10 @@ The NotImplemented Object This object is returned from comparisons and binary operations when they are asked to operate on types they don't support. See :ref:`comparisons` for more -information. There is exactly one ``NotImplemented`` object. -``type(NotImplemented)()`` produces the singleton instance. +information. There is exactly one :data:`NotImplemented` object. +:code:`type(NotImplemented)()` produces the singleton instance. -It is written as ``NotImplemented``. +It is written as :code:`NotImplemented`. .. _typesinternal: diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index b0a5d96c38d375..1f25a16f544da8 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2143,10 +2143,10 @@ to change the default. Methods and their defaults: -* ``__lt__``: ``NotImplemented`` -* ``__gt__``: ``NotImplemented`` -* ``__le__``: ``NotImplemented`` -* ``__ge__``: ``NotImplemented`` +* ``__lt__``: :data:`NotImplemented` +* ``__gt__``: :data:`!NotImplemented` +* ``__le__``: :data:`!NotImplemented` +* ``__ge__``: :data:`!NotImplemented` * ``__int__``: ``1`` * ``__contains__``: ``False`` * ``__len__``: ``0`` diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 6438abbba10e37..1aef2668e82cd1 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -159,7 +159,7 @@ NotImplemented .. index:: pair: object; NotImplemented This type has a single value. There is a single object with this value. This -object is accessed through the built-in name ``NotImplemented``. Numeric methods +object is accessed through the built-in name :data:`NotImplemented`. Numeric methods and rich comparison methods should return this value if they do not implement the operation for the operands provided. (The interpreter will then try the reflected operation, or some other fallback, depending on the operator.) It @@ -170,7 +170,7 @@ See for more details. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While + Evaluating :data:`NotImplemented` in a boolean context is deprecated. While it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. It will raise a :exc:`TypeError` in a future version of Python. @@ -1787,7 +1787,7 @@ Basic customization ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls ``x.__ge__(y)``. - A rich comparison method may return the singleton ``NotImplemented`` if it does + A rich comparison method may return the singleton :data:`NotImplemented` if it does not implement the operation for a given pair of arguments. By convention, ``False`` and ``True`` are returned for a successful comparison. However, these methods can return any value, so if the comparison operator is used in a Boolean @@ -1795,10 +1795,10 @@ Basic customization :func:`bool` on the value to determine if the result is true or false. By default, ``object`` implements :meth:`__eq__` by using ``is``, returning - ``NotImplemented`` in the case of a false comparison: + :data:`NotImplemented` in the case of a false comparison: ``True if x is y else NotImplemented``. For :meth:`__ne__`, by default it delegates to :meth:`__eq__` and inverts the result unless it is - ``NotImplemented``. There are no other implied relationships among the + :data:`!NotImplemented`. There are no other implied relationships among the comparison operators or default implementations; for example, the truth of ``(x=`` 0. The return value may also be - :const:`NotImplemented`, which is treated the same as if the + :data:`NotImplemented`, which is treated the same as if the ``__length_hint__`` method didn't exist at all. This method is purely an optimization and is never required for correctness. @@ -2980,7 +2980,7 @@ left undefined. function is to be supported. If one of those methods does not support the operation with the supplied - arguments, it should return ``NotImplemented``. + arguments, it should return :data:`NotImplemented`. .. method:: object.__radd__(self, other) @@ -3010,7 +3010,7 @@ left undefined. types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is an instance of a class that has an :meth:`__rsub__` method, ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns - *NotImplemented*. + :data:`NotImplemented`. .. index:: pair: built-in function; pow @@ -3503,7 +3503,7 @@ An example of an asynchronous context manager class:: the behavior that ``None`` is not callable. .. [#] "Does not support" here means that the class has no such method, or - the method returns ``NotImplemented``. Do not set the method to + the method returns :data:`NotImplemented`. Do not set the method to ``None`` if you want to force fallback to the right operand's reflected method—that will instead have the opposite effect of explicitly *blocking* such fallback. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 50e0f97a6534af..00b57effd3e1c0 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1539,7 +1539,7 @@ built-in types. ``x == x`` are all false, while ``x != x`` is true. This behavior is compliant with IEEE 754. -* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that +* ``None`` and :data:`NotImplemented` are singletons. :PEP:`8` advises that comparisons for singletons should always be done with ``is`` or ``is not``, never the equality operators. diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 2a42664c02852c..5c99fbc503ba65 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1130,7 +1130,7 @@ changes, or look through the Subversion logs for all the details. (Added by Raymond Hettinger; :issue:`1818`.) Finally, the :class:`~collections.abc.Mapping` abstract base class now - returns :const:`NotImplemented` if a mapping is compared to + returns :data:`NotImplemented` if a mapping is compared to another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 9770b12c8af711..e35179a2d8e513 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -828,7 +828,7 @@ Other Language Changes :meth:`~object.__index__` method). (Contributed by Serhiy Storchaka in :issue:`37999`.) -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will +* If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. (Contributed by Alex Shkop in :issue:`38302`.) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index e07eda985d9bad..3dd400c3771ed2 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -872,7 +872,7 @@ multiple implementations of an operation that allows it to work with PEP written and implemented by Łukasz Langa. :func:`~functools.total_ordering` now supports a return value of -:const:`NotImplemented` from the underlying comparison function. (Contributed +:data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) A pure-python version of the :func:`~functools.partial` function is now in the diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index b24c13813be220..49d926b0edcd0f 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1126,7 +1126,7 @@ Changes in the Python API ``logging.getLogger(__name__)`` in some top-level module called ``'root.py'``. (Contributed by Vinay Sajip in :issue:`37742`.) -* Division handling of :class:`~pathlib.PurePath` now returns ``NotImplemented`` +* Division handling of :class:`~pathlib.PurePath` now returns :data:`NotImplemented` instead of raising a :exc:`TypeError` when passed something other than an instance of ``str`` or :class:`~pathlib.PurePath`. This allows creating compatible classes that don't inherit from those mentioned types. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index c379b968c9885b..bad3528084897b 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -158,7 +158,7 @@ tests that are unrelated to :class:`ProcessPoolExecutor` on those platforms. .. nonce: hsCNgX .. section: Core and Builtins -If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator +If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index e8d4a02a11e0f9..754e782dfe661b 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -660,7 +660,7 @@ Karabas. .. section: Core and Builtins Parameter substitution of the union type with wrong types now raises -``TypeError`` instead of returning ``NotImplemented``. +``TypeError`` instead of returning :data:`NotImplemented`. .. From 91c3c64237f56bde9d1c1b8127fdcb02a112b5a4 Mon Sep 17 00:00:00 2001 From: Carl Bordum Hansen Date: Thu, 29 Feb 2024 21:56:04 +0100 Subject: [PATCH 63/82] gh-73580: Docs for tunnelling TLS through TLS (GH-22539) --- Doc/library/ssl.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index fa81c3f208cff7..ce8f3dd2d317c1 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1790,6 +1790,9 @@ to speed up repeated connections from the same clients. *session*, see :attr:`~SSLSocket.session`. + To wrap an :class:`SSLSocket` in another :class:`SSLSocket`, use + :meth:`SSLContext.wrap_bio`. + .. versionchanged:: 3.5 Always allow a server_hostname to be passed, even if OpenSSL does not have SNI. From 83c5ecdeec80fbd1f667f234f626c4154d40ebb5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 29 Feb 2024 21:24:05 +0000 Subject: [PATCH 64/82] gh-108051: Update versions found by find_python.bat and clarify readme (GH-116118) --- PCbuild/find_python.bat | 7 ++++++- PCbuild/readme.txt | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d3f62c93869003..af85f6d362466e 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -29,6 +29,9 @@ :begin_search @set PYTHON= +@rem If PYTHON_FOR_BUILD is set, use that +@if NOT "%PYTHON_FOR_BUILD%"=="" @(set PYTHON="%PYTHON_FOR_BUILD%") && (set _Py_Python_Source=found as PYTHON_FOR_BUILD) && goto :found + @rem If there is an active virtual env, use that one @if NOT "%VIRTUAL_ENV%"=="" (set PYTHON="%VIRTUAL_ENV%\Scripts\python.exe") & (set _Py_Python_Source=found in virtual env) & goto :found @@ -42,7 +45,9 @@ @if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 9)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@rem It is fine to add new versions to this list when they have released, +@rem but we do not use prerelease builds here. +@for %%p in (3.12 3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 387565515fa0b0..a7c2a68928ca84 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -226,12 +226,18 @@ directory. This script extracts all the external sub-projects from and https://github.com/python/cpython-bin-deps via a Python script called "get_external.py", located in this directory. -If Python 3.6 or later is not available via the "py.exe" launcher, the -path or command to use for Python can be provided in the PYTHON_FOR_BUILD -environment variable, or get_externals.bat will download the latest -version of NuGet and use it to download the latest "pythonx86" package -for use with get_external.py. Everything downloaded by these scripts is -stored in ..\externals (relative to this directory). +Everything downloaded by these scripts is stored in ..\externals +(relative to this directory), or the path specified by the EXTERNALS_DIR +environment variable. + +The path or command to use for Python can be provided with the +PYTHON_FOR_BUILD environment variable. If this is not set, an active +virtual environment will be used. If none is active, and HOST_PYTHON is +set to a recent enough version or "py.exe" is able to find a recent +enough version, those will be used. If all else fails, a copy of Python +will be downloaded from NuGet and extracted to the externals directory. +This will then be used for later builds (see PCbuild/find_python.bat +for the full logic). It is also possible to download sources from each project's homepage, though you may have to change folder names or pass the names to MSBuild From 41d5391c551e2012f13d56ff4bcc1df15c2821d3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 Feb 2024 21:32:50 +0000 Subject: [PATCH 65/82] gh-71052: Add test exclusions to support running the test suite on Android (#115918) --- Lib/test/support/__init__.py | 13 +++++++++++-- Lib/test/support/os_helper.py | 17 +++++++++++++++++ Lib/test/test_asyncio/test_subprocess.py | 4 ++-- Lib/test/test_compileall.py | 2 +- Lib/test/test_fcntl.py | 3 ++- Lib/test/test_interpreters/test_lifecycle.py | 1 + Lib/test/test_os.py | 11 ++++++++--- Lib/test/test_pathlib/test_pathlib.py | 2 +- Lib/test/test_posix.py | 12 ++++++------ Lib/test/test_pty.py | 8 +++----- Lib/test/test_venv.py | 11 +++++------ ...024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst | 1 + 12 files changed, 58 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 401b2ce1fe213c..14e3766a34377b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -532,24 +532,33 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not -# have subprocess or fork support. is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" has_fork_support = hasattr(os, "fork") and not ( + # WASM and Apple mobile platforms do not support subprocesses. is_emscripten or is_wasi or is_apple_mobile + + # Although Android supports fork, it's unsafe to call it from Python because + # all Android apps are multi-threaded. + or is_android ) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") has_subprocess_support = not ( + # WASM and Apple mobile platforms do not support subprocesses. is_emscripten or is_wasi or is_apple_mobile + + # Although Android supports subproceses, they're almost never useful in + # practice (see PEP 738). And most of the tests that use them are calling + # sys.executable, which won't work when Python is embedded in an Android app. + or is_android ) def requires_subprocess(): diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 22787e32b5f3ab..ffa5fc52e9b2bd 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -198,6 +198,23 @@ def skip_unless_symlink(test): return test if ok else unittest.skip(msg)(test) +_can_hardlink = None + +def can_hardlink(): + global _can_hardlink + if _can_hardlink is None: + # Android blocks hard links using SELinux + # (https://stackoverflow.com/q/32365690). + _can_hardlink = hasattr(os, "link") and not support.is_android + return _can_hardlink + + +def skip_unless_hardlink(test): + ok = can_hardlink() + msg = "requires hardlink support" + return test if ok else unittest.skip(msg)(test) + + _can_xattr = None diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index f50a9ebc031ba8..890c0acc1720dc 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -13,6 +13,8 @@ from test import support from test.support import os_helper +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") if support.MS_WINDOWS: import msvcrt @@ -47,7 +49,6 @@ def _start(self, *args, **kwargs): self._proc.pid = -1 -@support.requires_subprocess() class SubprocessTransportTests(test_utils.TestCase): def setUp(self): super().setUp() @@ -111,7 +112,6 @@ def test_subprocess_repr(self): transport.close() -@support.requires_subprocess() class SubprocessMixin: def test_stdin_stdout(self): diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 0ec6013dc11e3e..14c2af19e3eb28 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -977,7 +977,7 @@ class CommandLineTestsNoSourceEpoch(CommandLineTestsBase, -@unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') +@os_helper.skip_unless_hardlink class HardlinkDedupTestsBase: # Test hardlink_dupes parameter of compileall.compile_dir() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 6d734d052454d3..8b4ed4a9e3a4fe 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -129,7 +129,8 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - platform.machine().startswith('arm') and platform.system() == 'Linux', + any(platform.machine().startswith(name) for name in {"arm", "aarch"}) + and platform.system() in {"Linux", "Android"}, "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index 67b6f439c3191f..becf003e2e5f20 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -164,6 +164,7 @@ def test_sys_path_0(self): class FinalizationTests(TestBase): + @support.requires_subprocess() def test_gh_109793(self): # Make sure finalization finishes and the correct error code # is reported, even when subinterpreters get cleaned up at the end. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2372ac4c21efd9..3b2d5fccc30f3c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1592,6 +1592,9 @@ def test_yields_correct_dir_fd(self): @unittest.skipIf( support.is_emscripten, "Cannot dup stdout on Emscripten" ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) def test_fd_leak(self): # Since we're opening a lot of FDs, we must be careful to avoid leaks: # we both check that calling fwalk() a large number of times doesn't @@ -2492,8 +2495,10 @@ def test_listdir(self): # test listdir without arguments current_directory = os.getcwd() try: - os.chdir(os.sep) - self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + # The root directory is not readable on Android, so use a directory + # we created ourselves. + os.chdir(self.dir) + self.assertEqual(set(os.listdir()), expected) finally: os.chdir(current_directory) @@ -4838,7 +4843,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): os.name == 'nt') def test_attributes(self): - link = hasattr(os, 'link') + link = os_helper.can_hardlink() symlink = os_helper.can_symlink() dirname = os.path.join(self.path, "dir") diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index c0dcf314da4bfc..7e44ae61a5eba7 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -796,7 +796,7 @@ def test_rmdir(self): self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") + @os_helper.skip_unless_hardlink def test_hardlink_to(self): P = self.cls(self.base) target = P / 'fileA' diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a45f620e18dc1d..2706d5eb6d9830 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1270,9 +1270,10 @@ def test_get_and_set_scheduler_and_param(self): self.assertIn(mine, possible_schedulers) try: parent = posix.sched_getscheduler(os.getppid()) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + # POSIX specifies EPERM, but Android returns EACCES. Both errno + # values are mapped to PermissionError. + pass else: self.assertIn(parent, possible_schedulers) self.assertRaises(OSError, posix.sched_getscheduler, -1) @@ -1287,9 +1288,8 @@ def test_get_and_set_scheduler_and_param(self): try: posix.sched_setscheduler(0, mine, param) posix.sched_setparam(0, param) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + pass self.assertRaises(OSError, posix.sched_setparam, -1, param) self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 3f2bac0155fd9e..dee94533c74549 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,7 +1,6 @@ -import sys import unittest from test.support import ( - is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -9,9 +8,8 @@ # Skip these tests if termios is not available import_module('termios') -# Skip tests on WASM platforms, plus iOS/tvOS/watchOS -if is_apple_mobile or is_emscripten or is_wasi: - raise unittest.SkipTest(f"pty tests not required on {sys.platform}") +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("pty is not available on this platform") import errno import os diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index ba31beb81e80b0..f60c662d322e38 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -19,8 +19,9 @@ import tempfile from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_apple_mobile, is_emscripten, - is_wasi, requires_venv_with_pip, TEST_HOME_DIR, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, + requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest @@ -39,10 +40,8 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -# Skip tests on WASM platforms, plus iOS/tvOS/watchOS -if is_apple_mobile or is_emscripten or is_wasi: - raise unittest.SkipTest(f"venv tests not required on {sys.platform}") - +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() def check_output(cmd, encoding=None): diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst b/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst new file mode 100644 index 00000000000000..9d3467ca7e7d40 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst @@ -0,0 +1 @@ +Add test exclusions to support running the test suite on Android. From 556749c3e33f7787da149f75ca1702b80383bead Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 29 Feb 2024 13:38:50 -0800 Subject: [PATCH 66/82] gh-112075: Avoid locking shared keys on every assignment (#116087) --- Objects/dictobject.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 5016e255f70ef9..9b8fc4a958f7ad 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1597,19 +1597,11 @@ insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) } static Py_ssize_t -insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name) +insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name, Py_hash_t hash) { assert(PyUnicode_CheckExact(name)); ASSERT_KEYS_LOCKED(keys); - Py_hash_t hash = unicode_get_hash(name); - if (hash == -1) { - hash = PyUnicode_Type.tp_hash(name); - if (hash == -1) { - PyErr_Clear(); - return DKIX_EMPTY; - } - } Py_ssize_t ix = unicodekeys_lookup_unicode(keys, name, hash); if (ix == DKIX_EMPTY) { if (keys->dk_usable <= 0) { @@ -6692,8 +6684,25 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { - LOCK_KEYS(keys); - ix = insert_into_splitdictkeys(keys, name); + Py_hash_t hash = unicode_get_hash(name); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(name); + assert(hash != -1); + } + +#ifdef Py_GIL_DISABLED + // Try a thread-safe lookup to see if the index is already allocated + ix = unicodekeys_lookup_unicode_threadsafe(keys, name, hash); + if (ix == DKIX_EMPTY) { + // Lock keys and do insert + LOCK_KEYS(keys); + ix = insert_into_splitdictkeys(keys, name, hash); + UNLOCK_KEYS(keys); + } +#else + ix = insert_into_splitdictkeys(keys, name, hash); +#endif + #ifdef Py_STATS if (ix == DKIX_EMPTY) { if (PyUnicode_CheckExact(name)) { @@ -6709,7 +6718,6 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, } } #endif - UNLOCK_KEYS(keys); } if (ix == DKIX_EMPTY) { PyObject *dict = make_dict_from_instance_attributes( From ccfc042bbf31e53c44b8aae444afd8365b798422 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 29 Feb 2024 13:39:50 -0800 Subject: [PATCH 67/82] gh-87115: Set `__main__.__spec__` to `None` in pdb (#116141) --- Lib/pdb.py | 1 + Lib/test/test_pdb.py | 12 ++++++++++++ .../2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst | 1 + 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 0754e8b628cf57..519c1ccd5640a1 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -188,6 +188,7 @@ def namespace(self): __name__='__main__', __file__=self, __builtins__=__builtins__, + __spec__=None, ) @property diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2b0795cdad707e..3dd275caf43200 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2695,6 +2695,18 @@ def bœr(): ('bœr', 2), ) + def test_spec(self): + # Test that __main__.__spec__ is set to None when running a script + script = """ + import __main__ + print(__main__.__spec__) + """ + + commands = "continue" + + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn('None', stdout) + def test_find_function_first_executable_line(self): code = textwrap.dedent("""\ def foo(): pass diff --git a/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst b/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst new file mode 100644 index 00000000000000..844340583cd456 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst @@ -0,0 +1 @@ +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` From fa1d675309c6a08b0833cf25cffe476c6166aba3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 Feb 2024 21:58:20 +0000 Subject: [PATCH 68/82] gh-71052: Fix several Android build issues (#115955) This change is part of the work on PEP-738: Adding Android as a supported platform. * Remove the "1.0" suffix from libpython's filename on Android, which would prevent Gradle from packaging it into an app. * Simplify the build command in the Makefile so that libpython always gets given an SONAME with the `-Wl-h` argument, even if the SONAME is identical to the actual filename. * Disable a number of functions on Android which can be compiled and linked against, but always fail at runtime. As a result, the native _multiprocessing module is no longer built for Android. * gh-115390 (bee7bb331) added some pre-determined results to the configure script for things that can't be autodetected when cross-compiling; this change adds Android to these where appropriate. * Add a couple more pre-determined results for Android, and making them cover iOS as well. This means the --enable-ipv6 configure option will no longer be required on either platform. --- Lib/ctypes/__init__.py | 2 + Makefile.pre.in | 6 +-- ...4-02-26-14-54-58.gh-issue-71052.XvFay1.rst | 1 + Modules/_multiprocessing/multiprocessing.c | 2 +- configure | 47 ++++++++++++++--- configure.ac | 50 +++++++++++++++---- 6 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 141142a57dcb3e..d54ee05b15f5bd 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -468,6 +468,8 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) +elif hasattr(_sys, "getandroidapilevel"): + pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) else: diff --git a/Makefile.pre.in b/Makefile.pre.in index 7533a49b4392f0..ee65ecd918ce2a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -859,11 +859,9 @@ $(LIBRARY): $(LIBRARY_OBJS) $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) - if test $(INSTSONAME) != $(LDLIBRARY); then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ - else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ fi libpython3.so: libpython$(LDVERSION).so diff --git a/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst b/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst new file mode 100644 index 00000000000000..bda91335814936 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst @@ -0,0 +1 @@ +Fix several Android build issues diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2e6d8eb68c0243..1f6ab718a36984 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -181,7 +181,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} diff --git a/configure b/configure index f431c5dd15ec4a..45f232164e7094 100755 --- a/configure +++ b/configure @@ -7423,7 +7423,13 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h LDLIBRARY='libpython$(LDVERSION).so' BLDLIBRARY='-L. -lpython$(LDVERSION)' RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + if test "$with_pydebug" != yes then PY3LIBRARY=libpython3.so @@ -13726,7 +13732,14 @@ then : else $as_nop if test "$cross_compiling" = yes then : + +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else ac_cv_aligned_required=yes +fi else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17529,6 +17542,23 @@ else printf "%s\n" "$MACHDEP_OBJS" >&6; } fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid + setresuid setreuid setuid" + + # These functions are unimplemented and always return an error. + unimplemented_funcs="sem_open sem_unlink" + + for name in $privileged_funcs $unimplemented_funcs; do + as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` + + eval "$as_func_var=no" + + done +fi + # checks for library functions ac_fn_c_check_func "$LINENO" "accept4" "ac_cv_func_accept4" if test "x$ac_cv_func_accept4" = xyes @@ -22012,10 +22042,11 @@ fi done -# On iOS, clock_settime can be linked (so it is found by -# configure), but it raises a runtime error if used because apps can't change -# the clock. Force the symbol off. -if test "$ac_sys_system" != "iOS" ; then +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then for ac_func in clock_settime do : @@ -22295,7 +22326,9 @@ else $as_nop if test "$cross_compiling" = yes then : -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -26968,7 +27001,7 @@ CPPFLAGS=$ac_save_cppflags { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for device files" >&5 printf "%s\n" "$as_me: checking for device files" >&6;} -if test "$ac_sys_system" = "iOS" ; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no else diff --git a/configure.ac b/configure.ac index a5abb06a829413..b7671facc65a8a 100644 --- a/configure.ac +++ b/configure.ac @@ -1443,7 +1443,13 @@ if test $enable_shared = "yes"; then LDLIBRARY='libpython$(LDVERSION).so' BLDLIBRARY='-L. -lpython$(LDVERSION)' RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + if test "$with_pydebug" != yes then PY3LIBRARY=libpython3.so @@ -3675,7 +3681,14 @@ int main(void) }]])], [ac_cv_aligned_required=no], [ac_cv_aligned_required=yes], -[ac_cv_aligned_required=yes]) +[ +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else + ac_cv_aligned_required=yes +fi]) ]) if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], @@ -4872,6 +4885,22 @@ else AC_MSG_RESULT([$MACHDEP_OBJS]) fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid + setresuid setreuid setuid" + + # These functions are unimplemented and always return an error. + unimplemented_funcs="sem_open sem_unlink" + + for name in $privileged_funcs $unimplemented_funcs; do + AS_VAR_PUSHDEF([func_var], [ac_cv_func_$name]) + AS_VAR_SET([func_var], [no]) + AS_VAR_POPDEF([func_var]) + done +fi + # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ @@ -5216,10 +5245,11 @@ AC_CHECK_FUNCS([clock_getres], [], [ ]) ]) -# On iOS, clock_settime can be linked (so it is found by -# configure), but it raises a runtime error if used because apps can't change -# the clock. Force the symbol off. -if test "$ac_sys_system" != "iOS" ; then +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then AC_CHECK_FUNCS([clock_settime], [], [ AC_CHECK_LIB([rt], [clock_settime], [ AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) @@ -5371,7 +5401,9 @@ int main(void) [ac_cv_buggy_getaddrinfo=no], [ac_cv_buggy_getaddrinfo=yes], [ -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -6589,9 +6621,9 @@ CPPFLAGS=$ac_save_cppflags AC_MSG_NOTICE([checking for device files]) dnl NOTE: Inform user how to proceed with files when cross compiling. -dnl iOS cross-compile builds are predictable; they won't ever +dnl Some cross-compile builds are predictable; they won't ever dnl have /dev/ptmx or /dev/ptc, so we can set them explicitly. -if test "$ac_sys_system" = "iOS" ; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no else From c04a981ff414b101736688773dbe24f824ca32ce Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:09:06 -0800 Subject: [PATCH 69/82] Fix rendering of null character in ast.rst (#116080) --- Doc/library/ast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 5ed6eb63c0fb62..d10f3f275c0eaf 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2193,7 +2193,7 @@ and classes for traversing abstract syntax trees: is no guarantee that the parse (or success of the parse) is the same as when run on the Python version corresponding to ``feature_version``. - If source contains a null character ('\0'), :exc:`ValueError` is raised. + If source contains a null character (``\0``), :exc:`ValueError` is raised. .. warning:: Note that successfully parsing source code into an AST object doesn't From d01886c5c9e3a62921b304ba7e5145daaa56d3cf Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 1 Mar 2024 06:13:38 +0800 Subject: [PATCH 70/82] gh-115685: Type/values propagate for TO_BOOL in tier 2 (GH-115686) --- Include/internal/pycore_uop_ids.h | 41 ++++++++------- Include/internal/pycore_uop_metadata.h | 2 + Python/bytecodes.c | 5 ++ Python/executor_cases.c.h | 11 ++++ Python/optimizer_bytecodes.c | 61 +++++++++++++++++++++ Python/optimizer_cases.c.h | 73 ++++++++++++++++++++++---- 6 files changed, 163 insertions(+), 30 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3c133d97b2f03e..8f71eab44d914d 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -216,42 +216,43 @@ extern "C" { #define _POP_JUMP_IF_FALSE 402 #define _POP_JUMP_IF_TRUE 403 #define _POP_TOP POP_TOP +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 404 +#define _PUSH_FRAME 405 #define _PUSH_NULL PUSH_NULL #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 405 -#define _SEND 406 +#define _SAVE_RETURN_OFFSET 406 +#define _SEND 407 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 407 -#define _STORE_ATTR 408 -#define _STORE_ATTR_INSTANCE_VALUE 409 -#define _STORE_ATTR_SLOT 410 +#define _START_EXECUTOR 408 +#define _STORE_ATTR 409 +#define _STORE_ATTR_INSTANCE_VALUE 410 +#define _STORE_ATTR_SLOT 411 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 411 -#define _STORE_FAST_0 412 -#define _STORE_FAST_1 413 -#define _STORE_FAST_2 414 -#define _STORE_FAST_3 415 -#define _STORE_FAST_4 416 -#define _STORE_FAST_5 417 -#define _STORE_FAST_6 418 -#define _STORE_FAST_7 419 +#define _STORE_FAST 412 +#define _STORE_FAST_0 413 +#define _STORE_FAST_1 414 +#define _STORE_FAST_2 415 +#define _STORE_FAST_3 416 +#define _STORE_FAST_4 417 +#define _STORE_FAST_5 418 +#define _STORE_FAST_6 419 +#define _STORE_FAST_7 420 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 420 +#define _STORE_SUBSCR 421 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 421 +#define _TO_BOOL 422 #define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT @@ -262,12 +263,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 422 +#define _UNPACK_SEQUENCE 423 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 422 +#define MAX_UOP_ID 423 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 35340fe9ee1b63..7f921a6cd3f4c8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -232,6 +232,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, @@ -425,6 +426,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_FRAME] = "_POP_FRAME", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1d515098f6c7e9..095982d64f34d0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4043,6 +4043,11 @@ dummy_func( value = ptr; } + tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + Py_DECREF(pop); + value = ptr; + } + tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { value = Py_NewRef(ptr); null = NULL; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9ec1be9076a5a0..e84c33281ee640 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3759,6 +3759,17 @@ break; } + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + PyObject *pop; + PyObject *value; + pop = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + Py_DECREF(pop); + value = ptr; + stack_pointer[-1] = value; + break; + } + case _LOAD_CONST_INLINE_WITH_NULL: { PyObject *value; PyObject *null; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index aa19a50f09ff71..2b47381ec76db4 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -254,6 +254,67 @@ dummy_func(void) { } } + op(_TO_BOOL, (value -- res)) { + (void)value; + res = sym_new_type(ctx, &PyBool_Type); + OUT_OF_SPACE_IF_NULL(res); + } + + op(_TO_BOOL_BOOL, (value -- value)) { + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + if(!sym_set_type(value, &PyBool_Type)) { + goto hit_bottom; + } + } + } + + op(_TO_BOOL_INT, (value -- res)) { + if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { + PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) + ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyLong_Type)) { + goto hit_bottom; + } + } + + op(_TO_BOOL_LIST, (value -- res)) { + if(!sym_set_type(value, &PyList_Type)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + + op(_TO_BOOL_NONE, (value -- res)) { + if (sym_get_const(value) == Py_None) { + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + } + sym_set_const(value, Py_None); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); + } + + op(_TO_BOOL_STR, (value -- res)) { + if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { + PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyUnicode_Type)) { + goto hit_bottom; + } + } + op(_LOAD_CONST, (-- value)) { // There should be no LOAD_CONST. It should be all // replaced by peephole_opt. diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b34c57376df88e..9d7ebb80f62857 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -104,45 +104,90 @@ } case _TO_BOOL: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + value = stack_pointer[-1]; + (void)value; + res = sym_new_type(ctx, &PyBool_Type); + OUT_OF_SPACE_IF_NULL(res); stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { + _Py_UopsSymbol *value; + value = stack_pointer[-1]; + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + if(!sym_set_type(value, &PyBool_Type)) { + goto hit_bottom; + } + } break; } case _TO_BOOL_INT: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + value = stack_pointer[-1]; + if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { + PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) + ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyLong_Type)) { + goto hit_bottom; + } stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + value = stack_pointer[-1]; + if(!sym_set_type(value, &PyList_Type)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + value = stack_pointer[-1]; + if (sym_get_const(value) == Py_None) { + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + } + sym_set_const(value, Py_None); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { + _Py_UopsSymbol *value; _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + value = stack_pointer[-1]; + if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { + PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyUnicode_Type)) { + goto hit_bottom; + } stack_pointer[-1] = res; break; } @@ -1789,6 +1834,14 @@ break; } + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + value = sym_new_unknown(ctx); + if (value == NULL) goto out_of_space; + stack_pointer[-1] = value; + break; + } + case _LOAD_CONST_INLINE_WITH_NULL: { _Py_UopsSymbol *value; _Py_UopsSymbol *null; From 6a95676bb526261434dd068d6c49927c44d24a9b Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Thu, 29 Feb 2024 23:52:50 +0100 Subject: [PATCH 71/82] gh-115398: Expose Expat >=2.6.0 reparse deferral API (CVE-2023-52425) (GH-115623) Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding five new methods: - `xml.etree.ElementTree.XMLParser.flush` - `xml.etree.ElementTree.XMLPullParser.flush` - `xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` - `xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` - `xml.sax.expatreader.ExpatParser.flush` Based on the "flush" idea from https://github.com/python/cpython/pull/115138#issuecomment-1932444270 . ### Notes - Please treat as a security fix related to CVE-2023-52425. Includes code suggested-by: Snild Dolkow and by core dev Serhiy Storchaka. --- Doc/library/pyexpat.rst | 31 ++++++++ Doc/library/xml.etree.elementtree.rst | 29 +++++++ Doc/whatsnew/3.13.rst | 11 +++ Include/pyexpat.h | 4 +- Lib/test/test_pyexpat.py | 54 +++++++++++++ Lib/test/test_sax.py | 51 ++++++++++++ Lib/test/test_xml_etree.py | 79 +++++++++++++++---- Lib/xml/etree/ElementTree.py | 14 ++++ Lib/xml/sax/expatreader.py | 14 ++++ ...-02-18-03-14-40.gh-issue-115398.tzvxH8.rst | 8 ++ Misc/sbom.spdx.json | 4 +- Modules/_elementtree.c | 35 ++++++++ Modules/clinic/_elementtree.c.h | 19 ++++- Modules/clinic/pyexpat.c.h | 49 +++++++++++- Modules/expat/pyexpatns.h | 1 + Modules/pyexpat.c | 53 +++++++++++++ 16 files changed, 435 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index a6ae8fdaa4991c..c897ec9e47b7ca 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -196,6 +196,37 @@ XMLParser Objects :exc:`ExpatError` to be raised with the :attr:`code` attribute set to ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. +.. method:: xmlparser.SetReparseDeferralEnabled(enabled) + + .. warning:: + + Calling ``SetReparseDeferralEnabled(False)`` has security implications, + as detailed below; please make sure to understand these consequences + prior to using the ``SetReparseDeferralEnabled`` method. + + Expat 2.6.0 introduced a security mechanism called "reparse deferral" + where instead of causing denial of service through quadratic runtime + from reparsing large tokens, reparsing of unfinished tokens is now delayed + by default until a sufficient amount of input is reached. + Due to this delay, registered handlers may — depending of the sizing of + input chunks pushed to Expat — no longer be called right after pushing new + input to the parser. Where immediate feedback and taking over responsiblity + of protecting against denial of service from large tokens are both wanted, + calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral + for the current Expat parser instance, temporarily or altogether. + Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse + deferral. + + .. versionadded:: 3.13 + +.. method:: xmlparser.GetReparseDeferralEnabled() + + Returns whether reparse deferral is currently enabled for the given + Expat parser instance. + + .. versionadded:: 3.13 + + :class:`xmlparser` objects have the following attributes: diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 75a7915c15240d..19c7af452e2b71 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -166,6 +166,11 @@ data but would still like to have incremental parsing capabilities, take a look at :func:`iterparse`. It can be useful when you're reading a large XML document and don't want to hold it wholly in memory. +Where *immediate* feedback through events is wanted, calling method +:meth:`XMLPullParser.flush` can help reduce delay; +please make sure to study the related security notes. + + Finding interesting elements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1387,6 +1392,19 @@ XMLParser Objects Feeds data to the parser. *data* is encoded data. + + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + .. versionadded:: 3.13 + + :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method for each opening tag, its ``end(tag)`` method for each closing tag, and data is processed by method ``data(data)``. For further supported callback @@ -1448,6 +1466,17 @@ XMLPullParser Objects Feed the given bytes data to the parser. + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + .. versionadded:: 3.13 + .. method:: close() Signal the parser that the data stream is terminated. Unlike diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3a277d7ce1585f..d08c63e7b2c2c5 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -174,6 +174,17 @@ Other Language Changes (Contributed by Victor Stinner in :gh:`114570`.) +* Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) + by adding five new methods: + + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` + + (Contributed by Sebastian Pipping in :gh:`115623`.) + New Modules =========== diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 07020b5dc964cb..9824d099c3df7d 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -48,8 +48,10 @@ struct PyExpat_CAPI enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); int (*DefaultUnknownEncodingHandler)( void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); - /* might be none for expat < 2.1.0 */ + /* might be NULL for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.6.0 */ + XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); /* always add new stuff to the end! */ }; diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index d941a1a8f9ebc6..1d56ccd71cf962 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -755,5 +755,59 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ReparseDeferralTest(unittest.TestCase): + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index eda4e6a46df437..97e96668f85c8a 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -19,6 +19,7 @@ from io import BytesIO, StringIO import codecs import os.path +import pyexpat import shutil import sys from urllib.error import URLError @@ -1214,6 +1215,56 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + def test_flush_reparse_deferral_enabled(self): + if pyexpat.version_info < (2, 6, 0): + self.skipTest(f'Expat {pyexpat.version_info} does not support reparse deferral') + + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + + def test_flush_reparse_deferral_disabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + parser._parser.SetReparseDeferralEnabled(False) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + # ===== Locator support def test_expat_locator_noinfo(self): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c535d631bb646f..14df482ba6c207 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -121,10 +121,6 @@ """ -fails_with_expat_2_6_0 = (unittest.expectedFailure - if pyexpat.version_info >= (2, 6, 0) else - lambda test: test) - def checkwarnings(*filters, quiet=False): def decorator(test): def newtest(*args, **kwargs): @@ -1462,12 +1458,14 @@ def test_attlist_default(self): class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_events(self, parser, expected, max_events=None): self.assertEqual( @@ -1485,34 +1483,32 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - def test_simple_xml(self, chunk_size=None): + def test_simple_xml(self, chunk_size=None, flush=False): parser = ET.XMLPullParser() self.assert_event_tags(parser, []) - self._feed(parser, "\n", chunk_size) + self._feed(parser, "\n", chunk_size, flush) self.assert_event_tags(parser, []) self._feed(parser, "\n text\n", chunk_size) + self._feed(parser, ">\n", chunk_size, flush) self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "texttail\n", chunk_size) - self._feed(parser, "\n", chunk_size) + self._feed(parser, "texttail\n", chunk_size, flush) + self._feed(parser, "\n", chunk_size, flush) self.assert_event_tags(parser, [ ('end', 'element'), ('end', 'empty-element'), ]) - self._feed(parser, "\n", chunk_size) + self._feed(parser, "\n", chunk_size, flush) self.assert_event_tags(parser, [('end', 'root')]) self.assertIsNone(parser.close()) - @fails_with_expat_2_6_0 def test_simple_xml_chunk_1(self): - self.test_simple_xml(chunk_size=1) + self.test_simple_xml(chunk_size=1, flush=True) - @fails_with_expat_2_6_0 def test_simple_xml_chunk_5(self): - self.test_simple_xml(chunk_size=5) + self.test_simple_xml(chunk_size=5, flush=True) def test_simple_xml_chunk_22(self): self.test_simple_xml(chunk_size=22) @@ -1711,6 +1707,57 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + def test_flush_reparse_deferral_enabled(self): + if pyexpat.version_info < (2, 6, 0): + self.skipTest(f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index a37fead41b750e..9e15d34d22aa6c 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1320,6 +1320,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1726,6 +1731,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # -------------------------------------------------------------------- # C14N 2.0 diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index b9ad52692db8dd..ba3c1e98517429 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -214,6 +214,20 @@ def feed(self, data, isFinal=False): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst new file mode 100644 index 00000000000000..97b23936928d91 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst @@ -0,0 +1,8 @@ +Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding +five new methods: + +* ``xml.etree.ElementTree.XMLParser.flush`` +* ``xml.etree.ElementTree.XMLPullParser.flush`` +* ``xml.parsers.expat.xmlparser.GetReparseDeferralEnabled`` +* ``xml.parsers.expat.xmlparser.SetReparseDeferralEnabled`` +* ``xml.sax.expatreader.ExpatParser.flush`` diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index e28eaea81d6aae..27e6742292ac6d 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -132,11 +132,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "baa44fe4581895d42e8d5e83d8ce6a69b1c34dbe" + "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4" }, { "algorithm": "SHA256", - "checksumValue": "33a7b9ac8bf4571e23272cdf644c6f9808bd44c66b149e3c41ab3870d1888609" + "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567" } ], "fileName": "Modules/expat/pyexpatns.h" diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 54451081211654..edd2f88a4881c3 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -3894,6 +3894,40 @@ _elementtree_XMLParser_close_impl(XMLParserObject *self) } } +/*[clinic input] +_elementtree.XMLParser.flush + +[clinic start generated code]*/ + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self) +/*[clinic end generated code: output=42fdb8795ca24509 input=effbecdb28715949]*/ +{ + if (!_check_xmlparser(self)) { + return NULL; + } + + elementtreestate *st = self->state; + + if (EXPAT(st, SetReparseDeferralEnabled) == NULL) { + Py_RETURN_NONE; + } + + // NOTE: The Expat parser in the C implementation of ElementTree is not + // exposed to the outside; as a result we known that reparse deferral + // is currently enabled, or we would not even have access to function + // XML_SetReparseDeferralEnabled in the first place (which we checked + // for, a few lines up). + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_FALSE); + + PyObject *res = expat_parse(st, self, "", 0, XML_FALSE); + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_TRUE); + + return res; +} + /*[clinic input] _elementtree.XMLParser.feed @@ -4288,6 +4322,7 @@ static PyType_Spec treebuilder_spec = { static PyMethodDef xmlparser_methods[] = { _ELEMENTTREE_XMLPARSER_FEED_METHODDEF _ELEMENTTREE_XMLPARSER_CLOSE_METHODDEF + _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF _ELEMENTTREE_XMLPARSER__PARSE_WHOLE_METHODDEF _ELEMENTTREE_XMLPARSER__SETEVENTS_METHODDEF {NULL, NULL} diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 9622591a1aa855..10b2dd1c15f7fd 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -1169,6 +1169,23 @@ _elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored) return _elementtree_XMLParser_close_impl(self); } +PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_elementtree_XMLParser_flush, METH_NOARGS, _elementtree_XMLParser_flush__doc__}, + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self); + +static PyObject * +_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _elementtree_XMLParser_flush_impl(self); +} + PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__, "feed($self, data, /)\n" "--\n" @@ -1219,4 +1236,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=218ec9e6a889f796 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aed9f53eeb0404e0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index a5b93e68598204..343cb91b975038 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -8,6 +8,53 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, +"SetReparseDeferralEnabled($self, enabled, /)\n" +"--\n" +"\n" +"Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0."); + +#define PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF \ + {"SetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_SetReparseDeferralEnabled, METH_O, pyexpat_xmlparser_SetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled); + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int enabled; + + enabled = PyObject_IsTrue(arg); + if (enabled < 0) { + goto exit; + } + return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled); + +exit: + return return_value; +} + +PyDoc_STRVAR(pyexpat_xmlparser_GetReparseDeferralEnabled__doc__, +"GetReparseDeferralEnabled($self, /)\n" +"--\n" +"\n" +"Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0."); + +#define PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF \ + {"GetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_GetReparseDeferralEnabled, METH_NOARGS, pyexpat_xmlparser_GetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self); + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +{ + return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self); +} + PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "Parse($self, data, isfinal=False, /)\n" "--\n" @@ -498,4 +545,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=48c4296e43777df4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=892e48e41f9b6e4b input=a9049054013a1b77]*/ diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index d45d9b6c457159..8ee03ef0792815 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -108,6 +108,7 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 62cd262a7885e9..f04f96bc2f7601 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -7,6 +7,7 @@ #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() +#include #include // offsetof() #include "expat.h" #include "pyexpat.h" @@ -81,6 +82,12 @@ typedef struct { /* NULL if not enabled */ int buffer_size; /* Size of buffer, in XML_Char units */ int buffer_used; /* Buffer units in use */ + bool reparse_deferral_enabled; /* Whether to defer reparsing of + unfinished XML tokens; a de-facto cache of + what Expat has the authority on, for lack + of a getter API function + "XML_GetReparseDeferralEnabled" in Expat + 2.6.0 */ PyObject *intern; /* Dictionary to intern strings */ PyObject **handlers; } xmlparseobject; @@ -703,6 +710,40 @@ get_parse_result(pyexpat_state *state, xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) +/*[clinic input] +pyexpat.xmlparser.SetReparseDeferralEnabled + + enabled: bool + / + +Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled) +/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +{ +#if XML_COMBINED_VERSION >= 20600 + XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); + self->reparse_deferral_enabled = (bool)enabled; +#endif + Py_RETURN_NONE; +} + +/*[clinic input] +pyexpat.xmlparser.GetReparseDeferralEnabled + +Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self) +/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/ +{ + return PyBool_FromLong(self->reparse_deferral_enabled); +} + /*[clinic input] pyexpat.xmlparser.Parse @@ -1063,6 +1104,8 @@ static struct PyMethodDef xmlparse_methods[] = { #if XML_COMBINED_VERSION >= 19505 PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif + PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF + PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1158,6 +1201,11 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, self->ns_prefixes = 0; self->handlers = NULL; self->intern = Py_XNewRef(intern); +#if XML_COMBINED_VERSION >= 20600 + self->reparse_deferral_enabled = true; +#else + self->reparse_deferral_enabled = false; +#endif /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, @@ -2019,6 +2067,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20600 + capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; +#else + capi->SetReparseDeferralEnabled = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, From ca56c3a172e4ca08c278ddaf66ac2943a1d93288 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 1 Mar 2024 07:04:16 +0800 Subject: [PATCH 72/82] gh-103092: Add a mutex to make the PRNG state of rotatingtree concurrent-safe (#115301) --- .../2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst | 1 + Modules/_lsprof.c | 4 +--- Modules/rotatingtree.c | 9 +++++++++ Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst diff --git a/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst b/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst new file mode 100644 index 00000000000000..47701396c81737 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst @@ -0,0 +1 @@ +Isolate :mod:`_lsprof` (apply :pep:`687`). diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index f1cee7cb6f66bf..a76c3dea555783 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1005,9 +1005,7 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/rotatingtree.c b/Modules/rotatingtree.c index 07e08bc3167c0a..217e495b3d2a9d 100644 --- a/Modules/rotatingtree.c +++ b/Modules/rotatingtree.c @@ -1,3 +1,9 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_lock.h" #include "rotatingtree.h" #define KEY_LOWER_THAN(key1, key2) ((char*)(key1) < (char*)(key2)) @@ -10,17 +16,20 @@ static unsigned int random_value = 1; static unsigned int random_stream = 0; +static PyMutex random_mutex = {0}; static int randombits(int bits) { int result; + PyMutex_Lock(&random_mutex); if (random_stream < (1U << bits)) { random_value *= 1082527; random_stream = random_value; } result = random_stream & ((1<>= bits; + PyMutex_Unlock(&random_mutex); return result; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 45119664af4362..686a3d3160cc90 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -482,3 +482,4 @@ Modules/readline.c - sigwinch_ohandler - Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - +Modules/rotatingtree.c - random_mutex - From d7ddd90308324340855ddb9cc8dda2c1ee3a5944 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 29 Feb 2024 19:02:44 -0500 Subject: [PATCH 73/82] gh-115491: Fix Clang compiler warning (#116153) gh-115491: Fix compiler warning on macOS --- Objects/mimalloc/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c index b369a5ebcb23a7..e6286b54bedc14 100644 --- a/Objects/mimalloc/alloc.c +++ b/Objects/mimalloc/alloc.c @@ -27,7 +27,7 @@ terms of the MIT license. A copy of the license can be found in the file // ------------------------------------------------------ #if (MI_DEBUG>0) -static void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) { +static inline void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) { size_t offset = (size_t)page->debug_offset; if (offset < size) { memset((char*)block + offset, c, size - offset); From 2e94a6687c1a9750e9d2408a8dff0a422aeaf0e4 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 29 Feb 2024 17:02:12 -0800 Subject: [PATCH 74/82] gh-116099: Fix refcounting bug in `_queueobj_shared()` (gh-116164) This code decrefs `qidobj` twice in some paths. Since `qidobj` isn't used after the first `Py_DECREF()`, remove the others, and replace the `Py_DECREF()` with `Py_CLEAR()` to make it clear that the variable is dead. With this fix, `python -mtest test_interpreters -R 3:3 -mtest_queues` no longer fails with `_Py_NegativeRefcount: Assertion failed: object has negative ref count`. --- Modules/_xxinterpqueuesmodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 715bb766cac624..21ba817785cdfe 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -1189,7 +1189,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, .label = "queue ID", }; int res = idarg_int64_converter(qidobj, &converted); - Py_DECREF(qidobj); + Py_CLEAR(qidobj); if (!res) { assert(PyErr_Occurred()); return -1; @@ -1197,12 +1197,10 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, void *raw = _queueid_xid_new(converted.id); if (raw == NULL) { - Py_DECREF(qidobj); return -1; } _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, _queueobj_from_xid); - Py_DECREF(qidobj); _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); return 0; } From 339c8e1c13adc299a0e2e49c93067e7817692380 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 29 Feb 2024 18:53:32 -0800 Subject: [PATCH 75/82] gh-115999: Disable the specializing adaptive interpreter in free-threaded builds (#116013) For now, disable all specialization when the GIL might be disabled. --- Include/internal/pycore_code.h | 5 +++ Lib/test/test_capi/test_opt.py | 6 ++- Lib/test/test_generated_cases.py | 8 ++++ Lib/test/test_monitoring.py | 5 +++ Lib/test/test_opcache.py | 4 +- Lib/test/test_type_cache.py | 3 +- Python/ceval_macros.h | 8 ++++ Python/generated_cases.c.h | 55 ++++++++++++++++++++++++ Tools/cases_generator/tier1_generator.py | 5 +++ 9 files changed, 96 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 85536162132072..a4e64825743ada 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -245,7 +245,12 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); +#ifdef Py_GIL_DISABLED +// gh-115999 tracks progress on addressing this. +#define ENABLE_SPECIALIZATION 0 +#else #define ENABLE_SPECIALIZATION 1 +#endif /* Specialization functions */ diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d21765307f0f09..f4fcdea05e96bf 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -8,7 +8,7 @@ import _testinternalcapi -from test.support import script_helper +from test.support import script_helper, requires_specialization @contextlib.contextmanager @@ -31,6 +31,7 @@ def clear_executors(func): func.__code__ = func.__code__.replace() +@requires_specialization class TestOptimizerAPI(unittest.TestCase): def test_new_counter_optimizer_dealloc(self): @@ -133,6 +134,7 @@ def get_opnames(ex): return set(iter_opnames(ex)) +@requires_specialization class TestExecutorInvalidation(unittest.TestCase): def setUp(self): @@ -211,6 +213,7 @@ def f(): self.assertIsNone(exe) +@requires_specialization @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): @@ -572,6 +575,7 @@ def testfunc(n): self.assertLessEqual(count, 2) +@requires_specialization @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 6fcb5d58dd7f34..32c2c2fca05c4e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -350,12 +350,15 @@ def test_cache_effect(self): output = """ TARGET(OP) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; stack_pointer += -1; DISPATCH(); } @@ -399,6 +402,7 @@ def test_macro_instruction(self): INSTRUCTION_STATS(OP); PREDICTED(OP); _Py_CODEUNIT *this_instr = next_instr - 6; + (void)this_instr; PyObject *right; PyObject *left; PyObject *arg2; @@ -408,6 +412,7 @@ def test_macro_instruction(self): left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; op1(left, right); } /* Skip 2 cache entries */ @@ -415,6 +420,7 @@ def test_macro_instruction(self): arg2 = stack_pointer[-3]; { uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; res = op2(arg2, left, right); } stack_pointer[-3] = res; @@ -424,6 +430,7 @@ def test_macro_instruction(self): TARGET(OP1) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); PyObject *right; @@ -431,6 +438,7 @@ def test_macro_instruction(self): right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; op1(left, right); DISPATCH(); } diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 2fd822036bcff5..b02795903f6d17 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -9,6 +9,7 @@ import types import unittest import asyncio +from test.support import requires_specialization PAIR = (0,1) @@ -815,6 +816,9 @@ def func1(): self.check_events(func1, [("raise", KeyError)]) + # gh-116090: This test doesn't really require specialization, but running + # it without specialization exposes a monitoring bug. + @requires_specialization def test_implicit_stop_iteration(self): def gen(): @@ -963,6 +967,7 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) + @requires_specialization def test_no_unwind_for_shim_frame(self): class B: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 2b2783d57be8f4..5fb4b815c95d07 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,7 +4,7 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail +from test.support import threading_helper, check_impl_detail, requires_specialization # Skip this module on other interpreters, it is cpython specific: if check_impl_detail(cpython=False): @@ -506,6 +506,7 @@ def f(x, y): @threading_helper.requires_working_threading() +@requires_specialization class TestRacesDoNotCrash(unittest.TestCase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: @@ -1021,6 +1022,7 @@ def write(items): class C: pass +@requires_specialization class TestInstanceDict(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 58572c6f4d3157..e90e315c808361 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,7 +2,7 @@ import unittest import dis from test import support -from test.support import import_helper +from test.support import import_helper, requires_specialization try: from sys import _clear_type_cache except ImportError: @@ -94,6 +94,7 @@ class C: @support.cpython_only +@requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): _clear_type_cache() diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9674a7824a4690..b024b510cfda1f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -296,11 +296,19 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) +#ifdef Py_GIL_DISABLED +#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + /* gh-115999 tracks progress on addressing this. */ \ + static_assert(0, "The specializing interpreter is not yet thread-safe"); \ + } while (0); +#else #define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); +#endif #define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3312078e9a2d4d..4369d8c9df796b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -104,6 +104,7 @@ INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *rhs; PyObject *lhs; PyObject *res; @@ -112,6 +113,7 @@ lhs = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -419,6 +421,7 @@ INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *sub; PyObject *container; PyObject *res; @@ -427,6 +430,7 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -747,6 +751,7 @@ INSTRUCTION_STATS(CALL); PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -757,6 +762,7 @@ callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1176,6 +1182,7 @@ INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; PyObject *kwargs = NULL; PyObject *callargs; PyObject *func; @@ -1336,6 +1343,7 @@ INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; PyObject *kwnames; PyObject **args; PyObject *self_or_null; @@ -1937,6 +1945,7 @@ TARGET(CLEANUP_THROW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); PyObject *exc_value; @@ -1973,6 +1982,7 @@ INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *right; PyObject *left; PyObject *res; @@ -1981,6 +1991,7 @@ left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2324,6 +2335,7 @@ TARGET(END_ASYNC_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); PyObject *exc; @@ -2460,12 +2472,14 @@ INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *iter; PyObject *next; // _SPECIALIZE_FOR_ITER iter = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2884,6 +2898,7 @@ TARGET(INSTRUMENTED_CALL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); /* Skip 3 cache entries */ @@ -2909,6 +2924,7 @@ TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); int is_meth = PEEK(oparg + 2) != NULL; @@ -2925,6 +2941,7 @@ TARGET(INSTRUMENTED_END_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyObject *value; @@ -2947,6 +2964,7 @@ TARGET(INSTRUMENTED_END_SEND) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyObject *value; @@ -2968,6 +2986,7 @@ TARGET(INSTRUMENTED_FOR_ITER) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ @@ -3000,6 +3019,7 @@ TARGET(INSTRUMENTED_INSTRUCTION) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); int next_opcode = _Py_call_instrumentation_instruction( @@ -3016,6 +3036,7 @@ TARGET(INSTRUMENTED_JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -3026,6 +3047,7 @@ TARGET(INSTRUMENTED_JUMP_FORWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); @@ -3034,6 +3056,7 @@ TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); /* Skip 1 cache entry */ @@ -3045,6 +3068,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ @@ -3061,6 +3085,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ @@ -3083,6 +3108,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ @@ -3105,6 +3131,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ @@ -3121,6 +3148,7 @@ TARGET(INSTRUMENTED_RESUME) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; @@ -3151,6 +3179,7 @@ TARGET(INSTRUMENTED_RETURN_CONST) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); @@ -3174,6 +3203,7 @@ TARGET(INSTRUMENTED_RETURN_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; @@ -3198,6 +3228,7 @@ TARGET(INSTRUMENTED_YIELD_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; @@ -3261,6 +3292,7 @@ TARGET(JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -3387,6 +3419,7 @@ INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; + (void)this_instr; PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -3394,6 +3427,7 @@ owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -4083,11 +4117,13 @@ INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; PyObject *res; PyObject *null = NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -4279,6 +4315,7 @@ INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *class; PyObject *global_super; PyObject *self; @@ -4289,6 +4326,7 @@ global_super = stack_pointer[-3]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -4565,6 +4603,7 @@ TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; @@ -4582,6 +4621,7 @@ TARGET(POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyObject *value; @@ -4615,6 +4655,7 @@ TARGET(POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyObject *value; @@ -4648,6 +4689,7 @@ TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; @@ -4709,6 +4751,7 @@ TARGET(RAISE_VARARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; @@ -4738,6 +4781,7 @@ TARGET(RERAISE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); PyObject *exc; @@ -4779,6 +4823,7 @@ INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; assert(frame == tstate->current_frame); if (tstate->tracing == 0) { uintptr_t global_version = @@ -4916,6 +4961,7 @@ INSTRUCTION_STATS(SEND); PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *receiver; PyObject *v; PyObject *retval; @@ -4923,6 +4969,7 @@ receiver = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5103,12 +5150,14 @@ INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; PyObject *owner; PyObject *v; // _SPECIALIZE_STORE_ATTR owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5392,6 +5441,7 @@ INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *sub; PyObject *container; PyObject *v; @@ -5400,6 +5450,7 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5495,12 +5546,14 @@ INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; PyObject *value; PyObject *res; // _SPECIALIZE_TO_BOOL value = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5704,11 +5757,13 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index aba36ec74e5766..fb2ab931b1c108 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -87,6 +87,8 @@ def write_uop( out.emit( f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" ) + if inst.family is None: + out.emit(f"(void){cache.name};\n") offset += cache.size emit_tokens(out, uop, stack, inst) if uop.properties.stores_sp: @@ -131,8 +133,10 @@ def generate_tier1( needs_this = uses_this(inst) out.emit("\n") out.emit(f"TARGET({name}) {{\n") + unused_guard = "(void)this_instr;\n" if inst.family is None else "" if needs_this and not inst.is_target: out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + out.emit(unused_guard) else: out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") @@ -141,6 +145,7 @@ def generate_tier1( out.emit(f"PREDICTED({name});\n") if needs_this: out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) if inst.family is not None: out.emit( f"static_assert({inst.family.size} == {inst.size-1}" From 7895a61168aad4565a1d953104c9ec620e7c588f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 29 Feb 2024 22:46:33 -0800 Subject: [PATCH 76/82] gh-116098: Revert "gh-107674: Improve performance of `sys.settrace` (GH-114986)" (GH-116178) Revert "gh-107674: Improve performance of `sys.settrace` (GH-114986)" This reverts commit 0a61e237009bf6b833e13ac635299ee063377699. --- ...-02-04-07-45-29.gh-issue-107674.q8mCmi.rst | 1 - Python/bytecodes.c | 33 +++++++++---------- Python/ceval.c | 28 +++++++--------- Python/ceval_macros.h | 16 ++++----- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 33 +++++++++---------- Python/instrumentation.c | 4 ++- 7 files changed, 53 insertions(+), 64 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst deleted file mode 100644 index f9b96788bfad94..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst +++ /dev/null @@ -1 +0,0 @@ -Improved the performance of :func:`sys.settrace` significantly diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 095982d64f34d0..396a8f09f3feca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -143,23 +143,22 @@ dummy_func( tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); - if (tstate->tracing == 0) { - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - ERROR_IF(err, error); - next_instr = this_instr; - DISPATCH(); - } + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + ERROR_IF(err, error); + next_instr = this_instr; } - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; } - this_instr->op.code = RESUME_CHECK; } inst(RESUME_CHECK, (--)) { @@ -170,13 +169,13 @@ dummy_func( uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version && tstate->tracing == 0); + DEOPT_IF(eval_breaker != version); } inst(INSTRUMENTED_RESUME, (--)) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version && tstate->tracing == 0) { + if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } diff --git a/Python/ceval.c b/Python/ceval.c index 34f286e4e17eb8..f817f288903694 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -806,23 +806,17 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int { _Py_CODEUNIT *prev = frame->instr_ptr; _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b024b510cfda1f..22992aa09e1f38 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -347,16 +347,12 @@ do { \ // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ - if (tstate->tracing) {\ - next_instr = dest; \ - } else { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) { \ - next_instr = (dest)+1; \ - goto error; \ - } \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = (dest)+1; \ + goto error; \ } \ } while (0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e84c33281ee640..56ee93862743d5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20,7 +20,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version && tstate->tracing == 0) goto deoptimize; + if (eval_breaker != version) goto deoptimize; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4369d8c9df796b..e612c9e4c37632 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3153,7 +3153,7 @@ INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version && tstate->tracing == 0) { + if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } @@ -4825,23 +4825,22 @@ _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; assert(frame == tstate->current_frame); - if (tstate->tracing == 0) { - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - DISPATCH(); - } + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; } - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; } - this_instr->op.code = RESUME_CHECK; DISPATCH(); } @@ -4857,7 +4856,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version && tstate->tracing == 0, RESUME); + DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4b7d8b5a587504..6f1bc2e0a107df 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1156,13 +1156,15 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(tstate->tracing == 0); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; + if (tstate->tracing) { + goto done; + } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); From 0704166f9a9c6028314d50b493670e7862c968b3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 1 Mar 2024 10:03:32 +0200 Subject: [PATCH 77/82] gh-65824: Improve the "less" prompt in pydoc (GH-116050) Output the line number, the percentage and the help about how to get help or quit the pager. Inspired by the GNU man. --- Lib/pydoc.py | 11 ++++++++++- .../2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b0193b4a85164a..407c0205c7ab66 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1685,8 +1685,17 @@ def plain(text): def pipepager(text, cmd): """Page through text by feeding it to another program.""" import subprocess + env = os.environ.copy() + prompt_string = ( + ' ' + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace') + errors='backslashreplace', env=env) try: with proc.stdin as pipe: try: diff --git a/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst b/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst new file mode 100644 index 00000000000000..7bc6ced120a7be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst @@ -0,0 +1 @@ +Improve the ``less`` prompt in :mod:`pydoc`. From 19e16ce6f8920d512287577353c10836ae4ca6b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:24:27 +0000 Subject: [PATCH 78/82] build(deps-dev): bump types-psutil from 5.9.5.20240106 to 5.9.5.20240205 in /Tools (#116188) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index c0a63b40ff4155..33a0a5ba8c84cc 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.8.0 # needed for peg_generator: -types-psutil==5.9.5.20240106 +types-psutil==5.9.5.20240205 types-setuptools==69.0.0.20240125 From 7b4794319c56b6b00e852c745af50fd95bd1a550 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:27:35 +0000 Subject: [PATCH 79/82] build(deps): bump hypothesis from 6.97.4 to 6.98.15 in /Tools (#116189) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-hypothesis.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 064731a236ee86..1a45d1c431dd11 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.97.4 +hypothesis==6.98.15 From 4a630980328b67f0aba6a04c3950aa9dbd532895 Mon Sep 17 00:00:00 2001 From: Amethyst Reese Date: Fri, 1 Mar 2024 02:52:53 -0800 Subject: [PATCH 80/82] gh-116159: argparse: performance improvement parsing large number of options (#116162) When parsing positional vs optional arguments, the use of min with a list comprehension inside of a loop results in quadratic time based on the number of optional arguments given. When combined with use of prefix based argument files and a large number of optional flags, this can result in extremely slow parsing behavior. This replaces the min call with a simple loop with a short circuit to break at the next optional argument. Co-authored-by: Zsolt Dollenstein --- Lib/argparse.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index 4200dd5e334ad5..0dbdd67a82f391 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2153,10 +2153,11 @@ def consume_positionals(start_index): while start_index <= max_option_string_index: # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) + next_option_string_index = start_index + while next_option_string_index <= max_option_string_index: + if next_option_string_index in option_string_indices: + break + next_option_string_index += 1 if start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) From 8ab6c2775c4b2566477589cfc50fb64f020dc4de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:08:46 +0000 Subject: [PATCH 81/82] build(deps-dev): bump types-setuptools from 69.0.0.20240125 to 69.1.0.20240301 in /Tools (#116190) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 33a0a5ba8c84cc..2fb616e894a734 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.8.0 # needed for peg_generator: types-psutil==5.9.5.20240205 -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240301 From 59167c962efcae72e8d88aa4b33062ed3de4f120 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 1 Mar 2024 13:32:16 +0200 Subject: [PATCH 82/82] gh-101293: Fix support of custom callables and types in inspect.Signature.from_callable() (GH-115530) Support callables with the __call__() method and types with __new__() and __init__() methods set to class methods, static methods, bound methods, partial functions, and other types of methods and descriptors. Add tests for numerous types of callables and descriptors. --- Lib/inspect.py | 161 ++++---- Lib/test/test_inspect/test_inspect.py | 362 +++++++++++++++++- ...-02-15-19-11-49.gh-issue-101293.898b8l.rst | 4 + 3 files changed, 438 insertions(+), 89 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 9191d4761b2c76..8a2b2c96e993b5 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2039,15 +2039,17 @@ def _signature_get_user_defined_method(cls, method_name): named ``method_name`` and returns it only if it is a pure python function. """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return + if method_name == '__new__': + meth = getattr(cls, method_name, None) else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth + meth = getattr_static(cls, method_name, None) + if meth is None or isinstance(meth, _NonUserDefinedCallables): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return None + if method_name != '__new__': + meth = _descriptor_get(meth, cls) + return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): @@ -2492,6 +2494,15 @@ def _signature_from_function(cls, func, skip_bound_arg=True, __validate_parameters__=is_duck_function) +def _descriptor_get(descriptor, obj): + if isclass(descriptor): + return descriptor + get = getattr(type(descriptor), '__get__', _sentinel) + if get is _sentinel: + return descriptor + return get(descriptor, obj, type(obj)) + + def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, @@ -2600,7 +2611,6 @@ def _signature_from_callable(obj, *, wrapped_sig = _get_signature_of(obj.func) return _signature_get_partial(wrapped_sig, obj) - sig = None if isinstance(obj, type): # obj is a class or a metaclass @@ -2608,88 +2618,65 @@ def _signature_from_callable(obj, *, # in its metaclass call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: - sig = _get_signature_of(call) - else: - factory_method = None - new = _signature_get_user_defined_method(obj, '__new__') - init = _signature_get_user_defined_method(obj, '__init__') - - # Go through the MRO and see if any class has user-defined - # pure Python __new__ or __init__ method - for base in obj.__mro__: - # Now we check if the 'obj' class has an own '__new__' method - if new is not None and '__new__' in base.__dict__: - factory_method = new - break - # or an own '__init__' method - elif init is not None and '__init__' in base.__dict__: - factory_method = init - break + return _get_signature_of(call) - if factory_method is not None: - sig = _get_signature_of(factory_method) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'base' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, base, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return sigcls.from_callable(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) + new = _signature_get_user_defined_method(obj, '__new__') + init = _signature_get_user_defined_method(obj, '__init__') - elif not isinstance(obj, _NonUserDefinedCallables): - # An object with __call__ - # We also check that the 'obj' is not an instance of - # types.WrapperDescriptorType or types.MethodWrapperType to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') - if call is not None: + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + sig = _get_signature_of(new) + if skip_bound_arg: + sig = _signature_bound_method(sig) + return sig + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + return _get_signature_of(init) + + # At this point we know, that `obj` is a class, with no user- + # defined '__init__', '__new__', or class-level '__call__' + + for base in obj.__mro__[:-1]: + # Since '__text_signature__' is implemented as a + # descriptor that extracts text signature from the + # class docstring, if 'obj' is derived from a builtin + # class, its own '__text_signature__' may be 'None'. + # Therefore, we go through the MRO (except the last + # class in there, which is 'object') to find the first + # class with non-empty text signature. try: - sig = _get_signature_of(call) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig + text_sig = base.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + # If 'base' class has a __text_signature__ attribute: + # return a signature based on it + return _signature_fromstr(sigcls, base, text_sig) + + # No '__text_signature__' was found for the 'obj' class. + # Last option is to check if its '__init__' is + # object.__init__ or type.__init__. + if type not in obj.__mro__: + # We have a class (not metaclass), but no user-defined + # __init__ or __new__ for it + if (obj.__init__ is object.__init__ and + obj.__new__ is object.__new__): + # Return a signature of 'object' builtin. + return sigcls.from_callable(object) + else: + raise ValueError( + 'no signature found for builtin type {!r}'.format(obj)) - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) + else: + # An object with __call__ + call = getattr_static(type(obj), '__call__', None) + if call is not None: + call = _descriptor_get(call, obj) + return _get_signature_of(call) raise ValueError('callable {!r} is not supported by signature'.format(obj)) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 9dc37852ec205a..52cf68b93b85fa 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2928,9 +2928,12 @@ def p(name): return signature.parameters[name].default # This doesn't work now. # (We don't have a valid signature for "type" in 3.4) + class ThisWorksNow: + __call__ = type + # TODO: Support type. + self.assertEqual(ThisWorksNow()(1), int) + self.assertEqual(ThisWorksNow()('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "no signature found"): - class ThisWorksNow: - __call__ = type test_callable(ThisWorksNow()) # Regression test for issue #20786 @@ -3521,6 +3524,98 @@ def __init__(self, b): ((('a', ..., ..., "positional_or_keyword"),), ...)) + with self.subTest('classmethod'): + class CM(type): + @classmethod + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class CM(type): + __call__ = functools.partial(lambda x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class CM(type): + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class CM(type): + __call__ = ':'.join + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(['a', 'bc']), 'a:bc') + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class CM(type): + __call__ = (2).__pow__ + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(3), 8) + self.assertEqual(C(3, 7), 1) + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature((0).__pow__)) + class CM(type): def __new__(mcls, name, bases, dct, *, foo=1): return super().__new__(mcls, name, bases, dct) @@ -3582,6 +3677,169 @@ def __init__(self, b): ('bar', 2, ..., "keyword_only")), ...)) + def test_signature_on_class_with_init(self): + class C: + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + pass + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(lambda x, a: None, 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_class_with_new(self): + with self.subTest('FunctionType'): + class C: + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(lambda x, cls, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(lambda cls, x, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __new__ = str.__subclasscheck__ + + self.assertEqual(C(), False) + # TODO: Support BuiltinMethodType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + with self.subTest('MethodWrapperType'): + class C: + __new__ = type.__or__.__get__(int, type) + + self.assertEqual(C(), C | int) + # TODO: Support MethodWrapperType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + # TODO: Test ClassMethodDescriptorType + + with self.subTest('MethodDescriptorType'): + class C: + __new__ = type.__dict__['__subclasscheck__'] + + self.assertEqual(C(C), True) + self.assertEqual(self.signature(C), self.signature(C.__subclasscheck__)) + + with self.subTest('WrapperDescriptorType'): + class C: + __new__ = type.__or__ + + self.assertEqual(C(int), C | int) + # TODO: Support WrapperDescriptorType + # self.assertEqual(self.signature(C), self.signature(C.__or__)) + self.assertRaises(ValueError, self.signature, C) + def test_signature_on_subclass(self): class A: def __new__(cls, a=1, *args, **kwargs): @@ -3635,8 +3893,11 @@ class D(C): pass # Test meta-classes without user-defined __init__ or __new__ class C(type): pass class D(C): pass + self.assertEqual(C('A', (), {}).__name__, 'A') + # TODO: Support type. with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(C), None) + self.assertEqual(D('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) @@ -3686,6 +3947,103 @@ class Bar(Spam, Foo): ((('a', ..., ..., "positional_or_keyword"),), ...)) + with self.subTest('classmethod'): + class C: + @classmethod + def __call__(cls, a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __call__(a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class C: + __call__ = A().call + + self.assertEqual(C()(1), 1) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __call__ = functools.partial(lambda x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __call__ = ':'.join + + self.assertEqual(C()(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class C: + __call__ = (2).__pow__ + + self.assertEqual(C()(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('ClassMethodDescriptorType'): + class C(dict): + __call__ = dict.__dict__['fromkeys'] + + res = C()([1, 2], 3) + self.assertEqual(res, {1: 3, 2: 3}) + self.assertEqual(type(res), C) + self.assertEqual(self.signature(C()), self.signature(dict.fromkeys)) + + with self.subTest('MethodDescriptorType'): + class C(str): + __call__ = str.join + + self.assertEqual(C(':')(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('WrapperDescriptorType'): + class C(int): + __call__ = int.__pow__ + + self.assertEqual(C(2)(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('MemberDescriptorType'): + class C: + __slots__ = '__call__' + c = C() + c.__call__ = lambda a: a + self.assertEqual(c(1), 1) + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + def test_signature_on_wrapper(self): class Wrapper: def __call__(self, b): diff --git a/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst b/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst new file mode 100644 index 00000000000000..98365d2edbc4b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst @@ -0,0 +1,4 @@ +Support callables with the ``__call__()`` method and types with +``__new__()`` and ``__init__()`` methods set to class methods, static +methods, bound methods, partial functions, and other types of methods and +descriptors in :meth:`inspect.Signature.from_callable`.