Skip to content

Commit

Permalink
Restore compatibility with Python 3.11
Browse files Browse the repository at this point in the history 
* PyFrameObjects are now lazily allocated and no longer part of the
  PyThreadState. Therefore switch the _PyCFrame instead, but keep the
  PyFrameObject* for GC.

  python/cpython@ae0a2b7

* The recursion_depth is no longer stored directly in the PyThreadState
  and needs to be calculated from recursion_remaining and
  recursion_limit.

  python/cpython@b931077

* The exc_state was also simplified (no more exc_type and
  exc_traceback).

  python/cpython@396b583

* Finally the frame "data stack" needs to be saved and restored. How this
  was done in python-greenlet was used as a reference.

  python-greenlet/greenlet#280

* Add new test test_clean_callstack to check that the call stack from
  the creation of the Fiber doesn't leak into it (i.e.
  `tstate->cframe->current_frame = NULL;` in `stacklet__callback`;
  `tstate->frame = NULL;` in older Python versions).
  • Loading branch information
@jhasse
jhasse committed Jun 6, 2024
1 parent 7abe9c1 commit 4cb05a0
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 9 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/python-package.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
# would use windows-latest, but distutils tries to build against the version of MSVC that python was compiled with
# https://github.com/pypa/setuptools/blob/main/setuptools/_distutils/msvc9compiler.py#L403
os: [ "ubuntu-latest", "macos-12", "windows-2019" ]
python-version: ["3.7", "3.8", "3.9", "3.10"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
#os: [ "ubuntu-latest" ]
#python-version: ["3.8"]

Expand Down
60 changes: 52 additions & 8 deletions src/fibers.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,17 +221,32 @@ stacklet__callback(stacklet_handle h, void *arg)
/* set current thread state before starting this new Fiber */
tstate = PyThreadState_Get();
ASSERT(tstate != NULL);
tstate->exc_state.exc_value = NULL;
#if PY_MINOR_VERSION < 11
tstate->frame = NULL;
tstate->exc_state.exc_type = NULL;
tstate->exc_state.exc_value = NULL;
tstate->exc_state.exc_traceback = NULL;
#else
tstate->datastack_chunk = NULL;
tstate->datastack_top = NULL;
tstate->datastack_limit = NULL;
tstate->cframe->current_frame = NULL;
#endif
tstate->exc_state.previous_item = NULL;

self->ts.recursion_depth = tstate->recursion_depth;
self->ts.frame = NULL;
self->ts.exc_state.exc_type = NULL;
self->ts.exc_state.exc_value = NULL;
#if PY_MINOR_VERSION < 11
self->ts.recursion_depth = tstate->recursion_depth;
self->ts.exc_state.exc_type = NULL;
self->ts.exc_state.exc_traceback = NULL;
#else
self->ts.recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
self->ts.cframe = NULL;
self->ts.datastack_chunk = NULL;
self->ts.datastack_top = NULL;
self->ts.datastack_limit = NULL;
#endif
self->ts.exc_state.previous_item = NULL;

if (value == NULL) {
Expand Down Expand Up @@ -285,11 +300,21 @@ do_switch(Fiber *self, PyObject *value)
tstate = PyThreadState_Get();
ASSERT(tstate != NULL);
ASSERT(tstate->dict != NULL);
current->ts.exc_state.exc_value = tstate->exc_state.exc_value;
#if PY_MINOR_VERSION < 11
current->ts.recursion_depth = tstate->recursion_depth;
current->ts.frame = tstate->frame;
current->ts.exc_state.exc_type = tstate->exc_state.exc_type;
current->ts.exc_state.exc_value = tstate->exc_state.exc_value;
current->ts.exc_state.exc_traceback = tstate->exc_state.exc_traceback;
#else
current->ts.recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
current->ts.frame = PyThreadState_GetFrame(tstate);
Py_XDECREF(current->ts.frame);
current->ts.cframe = tstate->cframe;
current->ts.datastack_chunk = tstate->datastack_chunk;
current->ts.datastack_top = tstate->datastack_top;
current->ts.datastack_limit = tstate->datastack_limit;
#endif
current->ts.exc_state.previous_item = tstate->exc_state.previous_item;
ASSERT(current->stacklet_h == NULL);

Expand Down Expand Up @@ -328,17 +353,32 @@ do_switch(Fiber *self, PyObject *value)
}

/* restore state */
tstate->exc_state.exc_value = current->ts.exc_state.exc_value;
#if PY_MINOR_VERSION < 11
tstate->recursion_depth = current->ts.recursion_depth;
tstate->frame = current->ts.frame;
tstate->exc_state.exc_type = current->ts.exc_state.exc_type;
tstate->exc_state.exc_value = current->ts.exc_state.exc_value;
tstate->exc_state.exc_traceback = current->ts.exc_state.exc_traceback;
#else
tstate->recursion_remaining = tstate->recursion_limit - current->ts.recursion_depth;
tstate->cframe = current->ts.cframe;
tstate->datastack_chunk = current->ts.datastack_chunk;
tstate->datastack_top = current->ts.datastack_top;
tstate->datastack_limit = current->ts.datastack_limit;
#endif
tstate->exc_state.previous_item = current->ts.exc_state.previous_item;

current->ts.frame = NULL;
current->ts.exc_state.exc_type = NULL;
current->ts.exc_state.exc_value = NULL;
#if PY_MINOR_VERSION < 11
current->ts.exc_state.exc_type = NULL;
current->ts.exc_state.exc_traceback = NULL;
#else
current->ts.cframe = NULL;
current->ts.datastack_chunk = NULL;
current->ts.datastack_top = NULL;
current->ts.datastack_limit = NULL;
#endif
current->ts.exc_state.previous_item = NULL;

return result;
Expand Down Expand Up @@ -590,9 +630,11 @@ Fiber_tp_traverse(Fiber *self, visitproc visit, void *arg)
Py_VISIT(self->ts_dict);
Py_VISIT(self->parent);
Py_VISIT(self->ts.frame);
Py_VISIT(self->ts.exc_state.exc_type);
Py_VISIT(self->ts.exc_state.exc_value);
#if PY_MINOR_VERSION < 11
Py_VISIT(self->ts.exc_state.exc_type);
Py_VISIT(self->ts.exc_state.exc_traceback);
#endif
Py_VISIT(self->ts.exc_state.previous_item);

return 0;
Expand All @@ -609,9 +651,11 @@ Fiber_tp_clear(Fiber *self)
Py_CLEAR(self->ts_dict);
Py_CLEAR(self->parent);
Py_CLEAR(self->ts.frame);
Py_CLEAR(self->ts.exc_state.exc_type);
Py_CLEAR(self->ts.exc_state.exc_value);
#if PY_MINOR_VERSION < 11
Py_CLEAR(self->ts.exc_state.exc_type);
Py_CLEAR(self->ts.exc_state.exc_traceback);
#endif
Py_CLEAR(self->ts.exc_state.previous_item);

return 0;
Expand Down
6 changes: 6 additions & 0 deletions src/fibers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ typedef struct _fiber {
PyObject *args;
PyObject *kwargs;
struct {
#if PY_MINOR_VERSION >= 11
_PyCFrame *cframe;
_PyStackChunk *datastack_chunk;
PyObject **datastack_top;
PyObject **datastack_limit;
#endif
struct _frame *frame;
int recursion_depth;
_PyErr_StackItem exc_state;
Expand Down
14 changes: 14 additions & 0 deletions tests/test_fibers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import os
import sys
import traceback

import fibers
from fibers import Fiber, current
Expand Down Expand Up @@ -69,6 +70,19 @@ def f():
lst.append(4)
assert lst == list(range(5))

def test_clean_callstack(self):
lst = []

def f():
for line in traceback.format_stack():
lst.append(line)
f()
expected = [lst[-1]]
lst = []
g = Fiber(f)
g.switch()
assert lst == expected

def test_two_children(self):
lst = []

Expand Down

0 comments on commit 4cb05a0

Please sign in to comment.