From 8d3c477cba279ad0b551248b9e252b1c883ae4c2 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Thu, 6 Jun 2024 12:36:28 +0200 Subject: [PATCH] Restore compatibility with Python 3.11 * PyFrameObjects are now lazily allocated and no longer part of the PyThreadState. Therefore switch the _PyCFrame instead, but keep the PyFrameObject* for GC. https://github.com/python/cpython/commit/ae0a2b756255629140efcbe57fc2e714f0267aa3 * The recursion_depth is no longer stored directly in the PyThreadState and needs to be calculated from recursion_remaining and recursion_limit. https://github.com/python/cpython/commit/b9310773756f40f77e075f221a90dd41e6964efc * The exc_state was also simplified (no more exc_type and exc_traceback). https://github.com/python/cpython/commit/396b58345f81d4c8c5a52546d2288e666a1b9b8b * Finally the frame "data stack" needs to be saved and restored. How this was done in python-greenlet was used as a reference. https://github.com/python-greenlet/greenlet/pull/280 * Add new test test_clean_callstack to check that the call stack from the creation of the Fiber doesn't leak into it (i.e. `tstate->cframe->current_frame = NULL;` in `stacklet__callback`; `tstate->frame = NULL;` in older Python versions). --- .github/workflows/python-package.yml | 2 +- src/fibers.c | 60 ++++++++++++++++++++++++---- src/fibers.h | 6 +++ tests/test_fibers.py | 14 +++++++ 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ad56e10..d23c825 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ jobs: # would use windows-latest, but distutils tries to build against the version of MSVC that python was compiled with # https://github.com/pypa/setuptools/blob/main/setuptools/_distutils/msvc9compiler.py#L403 os: [ "ubuntu-latest", "macos-latest", "windows-2019" ] - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] #os: [ "ubuntu-latest" ] #python-version: ["3.8"] diff --git a/src/fibers.c b/src/fibers.c index d028056..76cc073 100644 --- a/src/fibers.c +++ b/src/fibers.c @@ -221,17 +221,32 @@ stacklet__callback(stacklet_handle h, void *arg) /* set current thread state before starting this new Fiber */ tstate = PyThreadState_Get(); ASSERT(tstate != NULL); + tstate->exc_state.exc_value = NULL; +#if PY_MINOR_VERSION < 11 tstate->frame = NULL; tstate->exc_state.exc_type = NULL; - tstate->exc_state.exc_value = NULL; tstate->exc_state.exc_traceback = NULL; +#else + tstate->datastack_chunk = NULL; + tstate->datastack_top = NULL; + tstate->datastack_limit = NULL; + tstate->cframe->current_frame = NULL; +#endif tstate->exc_state.previous_item = NULL; - self->ts.recursion_depth = tstate->recursion_depth; self->ts.frame = NULL; - self->ts.exc_state.exc_type = NULL; self->ts.exc_state.exc_value = NULL; +#if PY_MINOR_VERSION < 11 + self->ts.recursion_depth = tstate->recursion_depth; + self->ts.exc_state.exc_type = NULL; self->ts.exc_state.exc_traceback = NULL; +#else + self->ts.recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; + self->ts.cframe = NULL; + self->ts.datastack_chunk = NULL; + self->ts.datastack_top = NULL; + self->ts.datastack_limit = NULL; +#endif self->ts.exc_state.previous_item = NULL; if (value == NULL) { @@ -285,11 +300,21 @@ do_switch(Fiber *self, PyObject *value) tstate = PyThreadState_Get(); ASSERT(tstate != NULL); ASSERT(tstate->dict != NULL); + current->ts.exc_state.exc_value = tstate->exc_state.exc_value; +#if PY_MINOR_VERSION < 11 current->ts.recursion_depth = tstate->recursion_depth; current->ts.frame = tstate->frame; current->ts.exc_state.exc_type = tstate->exc_state.exc_type; - current->ts.exc_state.exc_value = tstate->exc_state.exc_value; current->ts.exc_state.exc_traceback = tstate->exc_state.exc_traceback; +#else + current->ts.recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; + current->ts.frame = PyThreadState_GetFrame(tstate); + Py_XDECREF(current->ts.frame); + current->ts.cframe = tstate->cframe; + current->ts.datastack_chunk = tstate->datastack_chunk; + current->ts.datastack_top = tstate->datastack_top; + current->ts.datastack_limit = tstate->datastack_limit; +#endif current->ts.exc_state.previous_item = tstate->exc_state.previous_item; ASSERT(current->stacklet_h == NULL); @@ -328,17 +353,32 @@ do_switch(Fiber *self, PyObject *value) } /* restore state */ + tstate->exc_state.exc_value = current->ts.exc_state.exc_value; +#if PY_MINOR_VERSION < 11 tstate->recursion_depth = current->ts.recursion_depth; tstate->frame = current->ts.frame; tstate->exc_state.exc_type = current->ts.exc_state.exc_type; - tstate->exc_state.exc_value = current->ts.exc_state.exc_value; tstate->exc_state.exc_traceback = current->ts.exc_state.exc_traceback; +#else + tstate->recursion_remaining = tstate->recursion_limit - current->ts.recursion_depth; + tstate->cframe = current->ts.cframe; + tstate->datastack_chunk = current->ts.datastack_chunk; + tstate->datastack_top = current->ts.datastack_top; + tstate->datastack_limit = current->ts.datastack_limit; +#endif tstate->exc_state.previous_item = current->ts.exc_state.previous_item; current->ts.frame = NULL; - current->ts.exc_state.exc_type = NULL; current->ts.exc_state.exc_value = NULL; +#if PY_MINOR_VERSION < 11 + current->ts.exc_state.exc_type = NULL; current->ts.exc_state.exc_traceback = NULL; +#else + current->ts.cframe = NULL; + current->ts.datastack_chunk = NULL; + current->ts.datastack_top = NULL; + current->ts.datastack_limit = NULL; +#endif current->ts.exc_state.previous_item = NULL; return result; @@ -590,9 +630,11 @@ Fiber_tp_traverse(Fiber *self, visitproc visit, void *arg) Py_VISIT(self->ts_dict); Py_VISIT(self->parent); Py_VISIT(self->ts.frame); - Py_VISIT(self->ts.exc_state.exc_type); Py_VISIT(self->ts.exc_state.exc_value); +#if PY_MINOR_VERSION < 11 + Py_VISIT(self->ts.exc_state.exc_type); Py_VISIT(self->ts.exc_state.exc_traceback); +#endif Py_VISIT(self->ts.exc_state.previous_item); return 0; @@ -609,9 +651,11 @@ Fiber_tp_clear(Fiber *self) Py_CLEAR(self->ts_dict); Py_CLEAR(self->parent); Py_CLEAR(self->ts.frame); - Py_CLEAR(self->ts.exc_state.exc_type); Py_CLEAR(self->ts.exc_state.exc_value); +#if PY_MINOR_VERSION < 11 + Py_CLEAR(self->ts.exc_state.exc_type); Py_CLEAR(self->ts.exc_state.exc_traceback); +#endif Py_CLEAR(self->ts.exc_state.previous_item); return 0; diff --git a/src/fibers.h b/src/fibers.h index dbf4c5d..264803c 100644 --- a/src/fibers.h +++ b/src/fibers.h @@ -30,6 +30,12 @@ typedef struct _fiber { PyObject *args; PyObject *kwargs; struct { +#if PY_MINOR_VERSION >= 11 + _PyCFrame *cframe; + _PyStackChunk *datastack_chunk; + PyObject **datastack_top; + PyObject **datastack_limit; +#endif struct _frame *frame; int recursion_depth; _PyErr_StackItem exc_state; diff --git a/tests/test_fibers.py b/tests/test_fibers.py index 4196f41..d2c82e1 100644 --- a/tests/test_fibers.py +++ b/tests/test_fibers.py @@ -6,6 +6,7 @@ import os import sys +import traceback import fibers from fibers import Fiber, current @@ -69,6 +70,19 @@ def f(): lst.append(4) assert lst == list(range(5)) + def test_clean_callstack(self): + lst = [] + + def f(): + for line in traceback.format_stack(): + lst.append(line) + f() + expected = [lst[-1]] + lst = [] + g = Fiber(f) + g.switch() + assert lst == expected + def test_two_children(self): lst = []