From 42d7186f639d3f172fc67e60c8c41ec58d4cccc4 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 6 Feb 2024 23:15:37 -0800 Subject: [PATCH 01/43] Basic prototype for frame proxy --- Include/cpython/frameobject.h | 6 + Include/cpython/pyframe.h | 2 + Include/internal/pycore_frame.h | 3 + Objects/frameobject.c | 339 +++++++++++++++++++++++++++++++- Python/frame.c | 2 + 5 files changed, 348 insertions(+), 4 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..dbbfbb5105ba7a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index c5adbbe4868f69..eeafbb17a56bad 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 0f9e7333cf1e1c..6742b1a188f418 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -61,6 +61,7 @@ typedef struct _PyInterpreterFrame { PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ + PyObject *f_extra_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ int stacktop; /* Offset of TOS from localsplus */ @@ -126,6 +127,7 @@ _PyFrame_Initialize( frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; + frame->f_extra_locals = PyDict_New(); frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); @@ -289,6 +291,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_globals = NULL; #endif frame->f_locals = NULL; + frame->f_extra_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a914c61aac2fd5..caca937331d5ba 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -13,9 +13,328 @@ #include "pycore_frame.h" #include "opcode.h" // EXTENDED_ARG +static int +_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg); #define OFF(x) offsetof(PyFrameObject, x) + +// Utilities + +// Returns borrowed reference or NULL +PyObject* framelocalproxy_getval(_PyInterpreterFrame* frame, PyCodeObject* co, int i) +{ + PyObject** fast = _PyFrame_GetLocalsArray(frame); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *value = fast[i]; + PyObject *cell = NULL; + + if (value == NULL) { + return NULL; + } + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { + // The cell was set when the frame was created from + // the function's closure. + assert(PyCell_Check(value)); + cell = value; + } + + if (cell != NULL) { + value = PyCell_GET(cell); + } + + if (value == NULL) { + return NULL; + } + + return value; +} + +// Type functions + +PyObject* framelocalsproxy_keys(PyObject* self, PyObject* __unused) +{ + PyObject* names = PyList_New(0); + _PyInterpreterFrame* frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; + PyCodeObject* co = _PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject* val = framelocalproxy_getval(frame, co, i); + if (val) { + PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyList_Append(names, name); + } + } + + // Iterate through the extra locals + Py_ssize_t i = 0; + PyObject* key = NULL; + PyObject* value = NULL; + + if (frame->f_extra_locals) { + assert(PyDict_Check(frame->f_extra_locals)); + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { + PyList_Append(names, key); + } + } + + return names; +} + +void framelocalsproxy_dealloc(PyObject *self) +{ + Py_DECREF(((PyFrameLocalsProxyObject*)self)->frame); + Py_TYPE(self)->tp_free(self); +} + +int framelocalsproxy_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject* frame = PyTuple_GET_ITEM(args, 0); + assert(PyFrame_Check(frame)); + Py_INCREF(frame); + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)frame; + + return 0; +} + +PyObject* framelocalsproxy_iter(PyObject *self) +{ + return PyObject_GetIter(framelocalsproxy_keys(self, NULL)); +} + +PyObject* framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +{ + if (PyFrameLocalsProxy_Check(other)) { + bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + if (op == Py_EQ) { + return PyBool_FromLong(result); + } else if (op == Py_NE) { + return PyBool_FromLong(!result); + } + } else if (PyDict_Check(other)) { + PyObject* dct = PyDict_New(); + PyObject* result = NULL; + PyDict_Update(dct, self); + result = PyObject_RichCompare(dct, other, op); + Py_DECREF(dct); + return result; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +// Methods + +PyObject* framelocalsproxy_items(PyObject* self, PyObject* __unused) +{ + PyObject* items = PyList_New(0); + _PyInterpreterFrame* frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; + PyCodeObject* co = _PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject* value = framelocalproxy_getval(frame, co, i); + + if (value) { + PyObject* pair = PyTuple_Pack(2, name, value); + PyList_Append(items, pair); + Py_DECREF(pair); + } + } + + // Iterate through the extra locals + Py_ssize_t j = 0; + PyObject* key = NULL; + PyObject* value = NULL; + + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject* pair = PyTuple_Pack(2, key, value); + PyList_Append(items, pair); + Py_DECREF(pair); + } + + return items; +} + +Py_ssize_t framelocalsproxy_length(PyObject* self) +{ + _PyInterpreterFrame *frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; + PyCodeObject *co = _PyFrame_GetCode(frame); + Py_ssize_t size = 0; + if (frame->f_extra_locals != NULL) { + assert(PyDict_Check(frame->f_extra_locals)); + size += PyDict_Size(frame->f_extra_locals); + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (framelocalproxy_getval(frame, co, i) != NULL) { + size++; + } + } + return size; +} + +PyObject* framelocalsproxy_getitem(PyObject* self, PyObject* key) +{ + _PyInterpreterFrame *frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; + PyCodeObject *co = _PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (_PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { + PyObject* value = framelocalproxy_getval(frame, co, i); + if (value == NULL) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%R' referenced before assignment", key); + return NULL; + } + return Py_NewRef(value); + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +int framelocalsproxy_setitem(PyObject* self, PyObject* key, PyObject* value) +{ + /* Merge locals into fast locals */ + _PyInterpreterFrame *frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; + PyObject **fast = _PyFrame_GetLocalsArray(frame); + PyCodeObject *co = _PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (_PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue != NULL && PyCell_Check(oldvalue)); + cell = oldvalue; + } else if (kind & CO_FAST_CELL && oldvalue != NULL) { + /* Same test as in PyFrame_FastToLocals() above. */ + if (PyCell_Check(oldvalue) && + _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { + // (likely) MAKE_CELL must have executed already. + cell = oldvalue; + } + // (unlikely) Otherwise, it must have been set to some + // initial value by an earlier call to PyFrame_LocalsToFast(). + } + if (cell != NULL) { + oldvalue = PyCell_GET(cell); + if (value != oldvalue) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue); + } + } else if (value != oldvalue) { + if (value == NULL) { + // Probably can't delete this, since the compiler's flow + // analysis may have already "proven" that it exists here: + const char *e = "assigning None to unbound local %R"; + if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, key)) { + // It's okay if frame_obj is NULL, just try anyways: + PyErr_WriteUnraisable((PyObject *)frame->frame_obj); + } + value = Py_NewRef(Py_None); + } + Py_XSETREF(fast[i], Py_NewRef(value)); + } + Py_XDECREF(value); + return 0; + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra != NULL) { + assert(PyDict_Check(extra)); + if (value == NULL) { + if (PyDict_DelItem(extra, key) < 0) { + return -1; + } + } else { + if (PyDict_SetItem(extra, key, value) < 0) { + return -1; + } + } + } + + return 0; +} + +static PyMappingMethods framelocalsproxy_as_mapping = { + framelocalsproxy_length, // mp_length + framelocalsproxy_getitem, // mp_subscript + framelocalsproxy_setitem, // mp_ass_subscript +}; + +static PyMethodDef framelocalsproxy_methods[] = { + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + NULL}, + {"keys", framelocalsproxy_keys, METH_NOARGS, + NULL}, + {"items", framelocalsproxy_items, METH_NOARGS, + NULL}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyFrameLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "FrameLocalsProxy", + sizeof(PyFrameLocalsProxyObject), + 0, + (destructor)framelocalsproxy_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &framelocalsproxy_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + framelocalsproxy_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + framelocalsproxy_iter, /* tp_iter */ + 0, /* tp_iternext */ + framelocalsproxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + framelocalsproxy_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ @@ -30,11 +349,21 @@ frame_getlocals(PyFrameObject *f, void *closure) return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); - if (locals) { - f->f_fast_as_locals = 1; + + PyCodeObject* co = PyFrame_GetCode(f); + + if (!(co->co_flags & CO_OPTIMIZED)) { + return Py_NewRef(f->f_frame->f_locals); } - return locals; + + PyFrameLocalsProxyObject* proxy = PyObject_New(PyFrameLocalsProxyObject, &PyFrameLocalsProxy_Type); + + if (proxy == NULL) { + return NULL; + } + + proxy->frame = (PyFrameObject*)Py_NewRef(f); + return (PyObject*)proxy; } int @@ -890,6 +1219,7 @@ frame_dealloc(PyFrameObject *f) frame->f_executable = NULL; Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); + Py_CLEAR(frame->f_extra_locals); PyObject **locals = _PyFrame_GetLocalsArray(frame); for (int i = 0; i < frame->stacktop; i++) { Py_CLEAR(locals[i]); @@ -927,6 +1257,7 @@ frame_tp_clear(PyFrameObject *f) } f->f_frame->stacktop = 0; Py_CLEAR(f->f_frame->f_locals); + Py_CLEAR(f->f_frame->f_extra_locals); return 0; } diff --git a/Python/frame.c b/Python/frame.c index ddf6ef6ba5465c..858d0442fff977 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -13,6 +13,7 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) { Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); + Py_VISIT(frame->f_extra_locals); Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ @@ -140,6 +141,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_XDECREF(frame->localsplus[i]); } Py_XDECREF(frame->f_locals); + Py_XDECREF(frame->f_extra_locals); Py_DECREF(frame->f_funcobj); } From 7eeab1b19e1a420a4346a4fafe8b69004af56aef Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 7 Feb 2024 17:30:07 -0800 Subject: [PATCH 02/43] Fix some lint and remove oprun check --- Objects/frameobject.c | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index caca937331d5ba..d5541e1d596bfb 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -13,9 +13,6 @@ #include "pycore_frame.h" #include "opcode.h" // EXTENDED_ARG -static int -_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg); - #define OFF(x) offsetof(PyFrameObject, x) @@ -71,7 +68,7 @@ PyObject* framelocalsproxy_keys(PyObject* self, PyObject* __unused) Py_ssize_t i = 0; PyObject* key = NULL; PyObject* value = NULL; - + if (frame->f_extra_locals) { assert(PyDict_Check(frame->f_extra_locals)); while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { @@ -225,13 +222,9 @@ int framelocalsproxy_setitem(PyObject* self, PyObject* key, PyObject* value) cell = oldvalue; } else if (kind & CO_FAST_CELL && oldvalue != NULL) { /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. + if (PyCell_Check(oldvalue)) { cell = oldvalue; } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). } if (cell != NULL) { oldvalue = PyCell_GET(cell); @@ -303,36 +296,36 @@ PyTypeObject PyFrameLocalsProxy_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - 0, /* tp_repr */ + 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ - &framelocalsproxy_as_mapping, /* tp_as_mapping */ + &framelocalsproxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - framelocalsproxy_richcompare, /* tp_richcompare */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + framelocalsproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - framelocalsproxy_iter, /* tp_iter */ + framelocalsproxy_iter, /* tp_iter */ 0, /* tp_iternext */ - framelocalsproxy_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ + framelocalsproxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - framelocalsproxy_init, /* tp_init */ + framelocalsproxy_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Del, /* tp_free */ }; static PyMemberDef frame_memberlist[] = { From 60e70e72958ecbcbefd22a23e145a0efdd9e73b6 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 7 Feb 2024 22:27:29 -0800 Subject: [PATCH 03/43] Not entirely work yet --- Include/internal/pycore_frame.h | 3 + Objects/frameobject.c | 109 +++++++++++--------------------- Python/ceval.c | 31 ++++++++- 3 files changed, 71 insertions(+), 72 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 6742b1a188f418..9bffd98f0ba785 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -237,6 +237,9 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); +PyObject * +_PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame); + int _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d5541e1d596bfb..2f3cfe6aea4b8a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -30,6 +30,7 @@ PyObject* framelocalproxy_getval(_PyInterpreterFrame* frame, PyCodeObject* co, i if (value == NULL) { return NULL; } + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { // The cell was set when the frame was created from // the function's closure. @@ -180,11 +181,9 @@ PyObject* framelocalsproxy_getitem(PyObject* self, PyObject* key) for (int i = 0; i < co->co_nlocalsplus; i++) { if (_PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { PyObject* value = framelocalproxy_getval(frame, co, i); - if (value == NULL) { - PyErr_Format(PyExc_UnboundLocalError, "local variable '%R' referenced before assignment", key); - return NULL; + if (value != NULL) { + return Py_NewRef(value); } - return Py_NewRef(value); } } @@ -1528,88 +1527,56 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) { - /* Merge fast locals into f->f_locals */ - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = frame->f_locals = PyDict_New(); - if (locals == NULL) { - return NULL; - } + PyCodeObject* co = _PyFrame_GetCode(frame); + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + if (!(co->co_flags & CO_OPTIMIZED)) { + return Py_NewRef(frame->f_locals); } - PyObject *hidden = NULL; - /* If include_hidden, "hidden" fast locals (from inlined comprehensions in - module/class scopes) will be included in the returned dict, but not in - frame->f_locals; the returned dict will be a modified copy. Non-hidden - locals will still be updated in frame->f_locals. */ - if (include_hidden) { - hidden = PyDict_New(); - if (hidden == NULL) { - return NULL; - } + PyFrameLocalsProxyObject* proxy = PyObject_New(PyFrameLocalsProxyObject, &PyFrameLocalsProxy_Type); + + if (proxy == NULL) { + return NULL; } - frame_init_get_vars(frame); + proxy->frame = (PyFrameObject*)Py_NewRef(f); + return (PyObject*)proxy; +} - PyCodeObject *co = _PyFrame_GetCode(frame); - for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value; // borrowed reference - if (!frame_get_var(frame, co, i, &value)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); +PyObject* +_PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) +{ + PyObject* hidden = PyDict_New(); + + if (hidden == NULL) { + return NULL; + } + + PyCodeObject* co = _PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - if (kind & CO_FAST_HIDDEN) { - if (include_hidden && value != NULL) { - if (PyObject_SetItem(hidden, name, value) != 0) { - goto error; - } - } + + if (!(kind & CO_FAST_HIDDEN)) { continue; } + + PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject* value = framelocalproxy_getval(frame, co, i); + if (value == NULL) { - if (PyObject_DelItem(locals, name) != 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - else { - goto error; - } - } - } - else { - if (PyObject_SetItem(locals, name, value) != 0) { - goto error; - } + continue; } - } - if (include_hidden && PyDict_Size(hidden)) { - PyObject *innerlocals = PyDict_New(); - if (innerlocals == NULL) { - goto error; - } - if (PyDict_Merge(innerlocals, locals, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - if (PyDict_Merge(innerlocals, hidden, 1) != 0) { - Py_DECREF(innerlocals); - goto error; + if (PyDict_SetItem(hidden, name, value) < 0) { + Py_DECREF(hidden); + return NULL; } - locals = innerlocals; } - else { - Py_INCREF(locals); - } - Py_CLEAR(hidden); - - return locals; - error: - Py_XDECREF(hidden); - return NULL; + return hidden; } diff --git a/Python/ceval.c b/Python/ceval.c index 4f208009086191..c75a29b1fcd11e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2458,13 +2458,42 @@ PyObject * _PyEval_GetFrameLocals(void) { PyThreadState *tstate = _PyThreadState_GET(); + PyObject* locals = NULL; _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); return NULL; } - return _PyFrame_GetLocals(current_frame, 1); + locals = _PyFrame_GetLocals(current_frame, 1); + if (locals == NULL) { + return NULL; + } + + if (PyFrameLocalsProxy_Check(locals)) { + PyObject* ret = PyDict_New(); + if (PyDict_Update(ret, locals)) { + Py_DECREF(ret); + return NULL; + } + Py_DECREF(locals); + return ret; + } + + if (PyMapping_Check(locals)) { + PyObject* hidden = _PyFrame_GetHiddenLocals(current_frame); + if (hidden == NULL) { + return NULL; + } + if (PyDict_Update(locals, hidden)) { + Py_DECREF(locals); + Py_DECREF(hidden); + return NULL; + } + return locals; + } + + return NULL; } PyObject * From 1454ce4b1c9fe7decd000d47bf2ba8a5ebc9d6ce Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 7 Feb 2024 22:45:40 -0800 Subject: [PATCH 04/43] Fix a bug --- Python/ceval.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index c75a29b1fcd11e..4b1563f353271d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2485,10 +2485,21 @@ _PyEval_GetFrameLocals(void) if (hidden == NULL) { return NULL; } - if (PyDict_Update(locals, hidden)) { - Py_DECREF(locals); + assert(PyDict_Check(hidden)); + if (PyDict_Size(hidden) > 0) { + PyObject* ret = PyDict_New(); + if (PyDict_Update(ret, locals)) { + Py_DECREF(ret); + Py_DECREF(hidden); + return NULL; + } + if (PyDict_Update(ret, hidden)) { + Py_DECREF(ret); + Py_DECREF(hidden); + return NULL; + } Py_DECREF(hidden); - return NULL; + return ret; } return locals; } From 0045274e196108bf88ba87ed03e283d8bf32e7d8 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 13 Feb 2024 11:36:04 -0800 Subject: [PATCH 05/43] Change code style and add GC --- Include/internal/pycore_frame.h | 4 +- Objects/frameobject.c | 188 +++++++++++++++++++------------- Python/frame.c | 2 - 3 files changed, 115 insertions(+), 79 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 9bffd98f0ba785..c6e4c4dfdd37c3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -26,6 +26,7 @@ struct _frame { char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ + PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -61,7 +62,6 @@ typedef struct _PyInterpreterFrame { PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ - PyObject *f_extra_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ int stacktop; /* Offset of TOS from localsplus */ @@ -127,7 +127,6 @@ _PyFrame_Initialize( frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; - frame->f_extra_locals = PyDict_New(); frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); @@ -294,7 +293,6 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_globals = NULL; #endif frame->f_locals = NULL; - frame->f_extra_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2f3cfe6aea4b8a..90317c70941a43 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -19,9 +19,10 @@ // Utilities // Returns borrowed reference or NULL -PyObject* framelocalproxy_getval(_PyInterpreterFrame* frame, PyCodeObject* co, int i) +PyObject * +framelocalproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) { - PyObject** fast = _PyFrame_GetLocalsArray(frame); + PyObject **fast = _PyFrame_GetLocalsArray(frame->f_frame); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); PyObject *value = fast[i]; @@ -51,24 +52,25 @@ PyObject* framelocalproxy_getval(_PyInterpreterFrame* frame, PyCodeObject* co, i // Type functions -PyObject* framelocalsproxy_keys(PyObject* self, PyObject* __unused) +PyObject * +framelocalsproxy_keys(PyObject *self, PyObject *__unused) { - PyObject* names = PyList_New(0); - _PyInterpreterFrame* frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; - PyCodeObject* co = _PyFrame_GetCode(frame); + PyObject *names = PyList_New(0); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject* val = framelocalproxy_getval(frame, co, i); + PyObject *val = framelocalproxy_getval(frame, co, i); if (val) { - PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyList_Append(names, name); } } // Iterate through the extra locals Py_ssize_t i = 0; - PyObject* key = NULL; - PyObject* value = NULL; + PyObject *key = NULL; + PyObject *value = NULL; if (frame->f_extra_locals) { assert(PyDict_Check(frame->f_extra_locals)); @@ -80,28 +82,61 @@ PyObject* framelocalsproxy_keys(PyObject* self, PyObject* __unused) return names; } -void framelocalsproxy_dealloc(PyObject *self) +void +framelocalsproxy_dealloc(PyObject *self) { - Py_DECREF(((PyFrameLocalsProxyObject*)self)->frame); + PyObject_GC_UnTrack(self); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); Py_TYPE(self)->tp_free(self); } -int framelocalsproxy_init(PyObject *self, PyObject *args, PyObject *kwds) +PyObject * +framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject* frame = PyTuple_GET_ITEM(args, 0); + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + PyFrameObject *frame = (PyFrameObject*)PyTuple_GET_ITEM(args, 0); assert(PyFrame_Check(frame)); - Py_INCREF(frame); - ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)frame; + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); + + if (frame->f_extra_locals == NULL) { + frame->f_extra_locals = PyDict_New(); + if (frame->f_extra_locals == NULL) { + Py_DECREF(frame); + Py_DECREF(self); + return NULL; + } + } + + return (PyObject *)self; +} + +static int +framelocalsproxy_tp_clear(PyFrameLocalsProxyObject *proxy) +{ + Py_CLEAR(proxy->frame); + return 0; +} + +int +framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); return 0; } -PyObject* framelocalsproxy_iter(PyObject *self) +PyObject * +framelocalsproxy_iter(PyObject *self) { return PyObject_GetIter(framelocalsproxy_keys(self, NULL)); } -PyObject* framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +PyObject * +framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) { if (PyFrameLocalsProxy_Check(other)) { bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; @@ -111,8 +146,8 @@ PyObject* framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) return PyBool_FromLong(!result); } } else if (PyDict_Check(other)) { - PyObject* dct = PyDict_New(); - PyObject* result = NULL; + PyObject *dct = PyDict_New(); + PyObject *result = NULL; PyDict_Update(dct, self); result = PyObject_RichCompare(dct, other, op); Py_DECREF(dct); @@ -124,18 +159,19 @@ PyObject* framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) // Methods -PyObject* framelocalsproxy_items(PyObject* self, PyObject* __unused) +PyObject * +framelocalsproxy_items(PyObject *self, PyObject *__unused) { - PyObject* items = PyList_New(0); - _PyInterpreterFrame* frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; - PyCodeObject* co = _PyFrame_GetCode(frame); + PyObject *items = PyList_New(0); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject* value = framelocalproxy_getval(frame, co, i); + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = framelocalproxy_getval(frame, co, i); if (value) { - PyObject* pair = PyTuple_Pack(2, name, value); + PyObject *pair = PyTuple_Pack(2, name, value); PyList_Append(items, pair); Py_DECREF(pair); } @@ -143,11 +179,11 @@ PyObject* framelocalsproxy_items(PyObject* self, PyObject* __unused) // Iterate through the extra locals Py_ssize_t j = 0; - PyObject* key = NULL; - PyObject* value = NULL; + PyObject *key = NULL; + PyObject *value = NULL; while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { - PyObject* pair = PyTuple_Pack(2, key, value); + PyObject *pair = PyTuple_Pack(2, key, value); PyList_Append(items, pair); Py_DECREF(pair); } @@ -155,11 +191,13 @@ PyObject* framelocalsproxy_items(PyObject* self, PyObject* __unused) return items; } -Py_ssize_t framelocalsproxy_length(PyObject* self) +Py_ssize_t +framelocalsproxy_length(PyObject *self) { - _PyInterpreterFrame *frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; - PyCodeObject *co = _PyFrame_GetCode(frame); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = PyFrame_GetCode(frame); Py_ssize_t size = 0; + if (frame->f_extra_locals != NULL) { assert(PyDict_Check(frame->f_extra_locals)); size += PyDict_Size(frame->f_extra_locals); @@ -173,14 +211,17 @@ Py_ssize_t framelocalsproxy_length(PyObject* self) return size; } -PyObject* framelocalsproxy_getitem(PyObject* self, PyObject* key) +PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) { - _PyInterpreterFrame *frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; - PyCodeObject *co = _PyFrame_GetCode(frame); + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - if (_PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { - PyObject* value = framelocalproxy_getval(frame, co, i); + // The name and the key might be interned so do a fast check first + if (PyTuple_GET_ITEM(co->co_localsplusnames, i) == key || + _PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { + PyObject *value = framelocalproxy_getval(frame, co, i); if (value != NULL) { return Py_NewRef(value); } @@ -201,15 +242,17 @@ PyObject* framelocalsproxy_getitem(PyObject* self, PyObject* key) return NULL; } -int framelocalsproxy_setitem(PyObject* self, PyObject* key, PyObject* value) +int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) { /* Merge locals into fast locals */ - _PyInterpreterFrame *frame = ((PyFrameLocalsProxyObject*)self)->frame->f_frame; - PyObject **fast = _PyFrame_GetLocalsArray(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject* co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - if (_PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { + if (PyTuple_GET_ITEM(co->co_localsplusnames, i) == key || + _PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); PyObject *oldvalue = fast[i]; @@ -220,7 +263,6 @@ int framelocalsproxy_setitem(PyObject* self, PyObject* key, PyObject* value) assert(oldvalue != NULL && PyCell_Check(oldvalue)); cell = oldvalue; } else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ if (PyCell_Check(oldvalue)) { cell = oldvalue; } @@ -238,7 +280,7 @@ int framelocalsproxy_setitem(PyObject* self, PyObject* key, PyObject* value) const char *e = "assigning None to unbound local %R"; if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, key)) { // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); + PyErr_WriteUnraisable((PyObject *)frame); } value = Py_NewRef(Py_None); } @@ -276,13 +318,13 @@ static PyMappingMethods framelocalsproxy_as_mapping = { }; static PyMethodDef framelocalsproxy_methods[] = { - {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, NULL}, - {"keys", framelocalsproxy_keys, METH_NOARGS, + {"keys", framelocalsproxy_keys, METH_NOARGS, NULL}, - {"items", framelocalsproxy_items, METH_NOARGS, + {"items", framelocalsproxy_items, METH_NOARGS, NULL}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; PyTypeObject PyFrameLocalsProxy_Type = { @@ -305,10 +347,10 @@ PyTypeObject PyFrameLocalsProxy_Type = { PyObject_GenericGetAttr, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ + framelocalsproxy_visit, /* tp_traverse */ + (inquiry)framelocalsproxy_tp_clear, /* tp_clear */ framelocalsproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ framelocalsproxy_iter, /* tp_iter */ @@ -321,12 +363,21 @@ PyTypeObject PyFrameLocalsProxy_Type = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - framelocalsproxy_init, /* tp_init */ + 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ + framelocalsproxy_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ }; +PyObject * +_PyFrameLocalsProxy_New(PyFrameObject *frame) +{ + PyObject* args = PyTuple_Pack(1, frame); + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + Py_DECREF(args); + return proxy; +} + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ @@ -348,14 +399,7 @@ frame_getlocals(PyFrameObject *f, void *closure) return Py_NewRef(f->f_frame->f_locals); } - PyFrameLocalsProxyObject* proxy = PyObject_New(PyFrameLocalsProxyObject, &PyFrameLocalsProxy_Type); - - if (proxy == NULL) { - return NULL; - } - - proxy->frame = (PyFrameObject*)Py_NewRef(f); - return (PyObject*)proxy; + return _PyFrameLocalsProxy_New(f); } int @@ -1211,7 +1255,6 @@ frame_dealloc(PyFrameObject *f) frame->f_executable = NULL; Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); - Py_CLEAR(frame->f_extra_locals); PyObject **locals = _PyFrame_GetLocalsArray(frame); for (int i = 0; i < frame->stacktop; i++) { Py_CLEAR(locals[i]); @@ -1219,6 +1262,7 @@ frame_dealloc(PyFrameObject *f) } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); PyObject_GC_Del(f); Py_XDECREF(co); Py_TRASHCAN_END; @@ -1229,6 +1273,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_extra_locals); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -1240,6 +1285,7 @@ static int frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); /* locals and stack */ PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); @@ -1249,7 +1295,6 @@ frame_tp_clear(PyFrameObject *f) } f->f_frame->stacktop = 0; Py_CLEAR(f->f_frame->f_locals); - Py_CLEAR(f->f_frame->f_extra_locals); return 0; } @@ -1378,6 +1423,7 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_extra_locals = NULL; return f; } @@ -1534,14 +1580,7 @@ _PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) return Py_NewRef(frame->f_locals); } - PyFrameLocalsProxyObject* proxy = PyObject_New(PyFrameLocalsProxyObject, &PyFrameLocalsProxy_Type); - - if (proxy == NULL) { - return NULL; - } - - proxy->frame = (PyFrameObject*)Py_NewRef(f); - return (PyObject*)proxy; + return _PyFrameLocalsProxy_New(f); } @@ -1549,6 +1588,7 @@ PyObject* _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) { PyObject* hidden = PyDict_New(); + PyFrameObject* f = _PyFrame_GetFrameObject(frame); if (hidden == NULL) { return NULL; @@ -1564,7 +1604,7 @@ _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) } PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject* value = framelocalproxy_getval(frame, co, i); + PyObject* value = framelocalproxy_getval(f, co, i); if (value == NULL) { continue; diff --git a/Python/frame.c b/Python/frame.c index 858d0442fff977..ddf6ef6ba5465c 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -13,7 +13,6 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) { Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); - Py_VISIT(frame->f_extra_locals); Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ @@ -141,7 +140,6 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_XDECREF(frame->localsplus[i]); } Py_XDECREF(frame->f_locals); - Py_XDECREF(frame->f_extra_locals); Py_DECREF(frame->f_funcobj); } From de73bc96bf3cfe48952e2e3913562f539877bdf2 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 13 Feb 2024 12:07:08 -0800 Subject: [PATCH 06/43] Clean up code --- Objects/frameobject.c | 57 +++++++++++++------------------------------ 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 90317c70941a43..9d5c493062bdcd 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -116,9 +116,9 @@ framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -framelocalsproxy_tp_clear(PyFrameLocalsProxyObject *proxy) +framelocalsproxy_tp_clear(PyObject *self) { - Py_CLEAR(proxy->frame); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); return 0; } @@ -329,44 +329,21 @@ static PyMethodDef framelocalsproxy_methods[] = { PyTypeObject PyFrameLocalsProxy_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "FrameLocalsProxy", - sizeof(PyFrameLocalsProxyObject), - 0, - (destructor)framelocalsproxy_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - &framelocalsproxy_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - framelocalsproxy_visit, /* tp_traverse */ - (inquiry)framelocalsproxy_tp_clear, /* tp_clear */ - framelocalsproxy_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - framelocalsproxy_iter, /* tp_iter */ - 0, /* tp_iternext */ - framelocalsproxy_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - framelocalsproxy_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ + .tp_name = "FrameLocalsProxy", + .tp_basicsize = sizeof(PyFrameLocalsProxyObject), + .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_as_mapping = &framelocalsproxy_as_mapping, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = framelocalsproxy_visit, + .tp_clear = framelocalsproxy_tp_clear, + .tp_richcompare = framelocalsproxy_richcompare, + .tp_iter = framelocalsproxy_iter, + .tp_methods = framelocalsproxy_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = framelocalsproxy_new, + .tp_free = PyObject_GC_Del, }; PyObject * From ca9239301aed9c9e223ce9e266e57265fe957766 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 29 Feb 2024 19:37:21 -0800 Subject: [PATCH 07/43] Disable all fast/local functions --- Include/internal/pycore_frame.h | 2 +- Objects/frameobject.c | 116 ++------------------------------ Python/ceval.c | 2 +- Python/intrinsics.c | 3 +- 4 files changed, 9 insertions(+), 114 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index c6e4c4dfdd37c3..66a133c7a6f5ad 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -234,7 +234,7 @@ int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); +_PyFrame_GetLocals(_PyInterpreterFrame *frame); PyObject * _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 9d5c493062bdcd..ca811696abef4e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -934,20 +934,6 @@ first_line_not_before(int *lines, int len, int line) return result; } -static bool -frame_is_cleared(PyFrameObject *frame) -{ - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->stacktop == 0) { - return true; - } - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_CLEARED; - } - return false; -} - static bool frame_is_suspended(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); @@ -1548,7 +1534,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) +_PyFrame_GetLocals(_PyInterpreterFrame *frame) { PyCodeObject* co = _PyFrame_GetCode(frame); PyFrameObject* f = _PyFrame_GetFrameObject(frame); @@ -1600,11 +1586,6 @@ _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) int _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { - PyObject *locals = _PyFrame_GetLocals(frame, 0); - if (locals == NULL) { - return -1; - } - Py_DECREF(locals); return 0; } @@ -1659,112 +1640,25 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - if (f == NULL) { - PyErr_BadInternalCall(); - return -1; - } - assert(!_PyFrame_IsIncomplete(f->f_frame)); - int err = _PyFrame_FastToLocalsWithError(f->f_frame); - if (err == 0) { - f->f_fast_as_locals = 1; - } - return err; + return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { - int res; - assert(!_PyFrame_IsIncomplete(f->f_frame)); - assert(!PyErr_Occurred()); - - res = PyFrame_FastToLocalsWithError(f); - if (res < 0) - PyErr_Clear(); + return; } void _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) { - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyCodeObject *co; - locals = frame->f_locals; - if (locals == NULL) { - return; - } - fast = _PyFrame_GetLocalsArray(frame); - co = _PyFrame_GetCode(frame); - - PyObject *exc = PyErr_GetRaisedException(); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; - } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } - else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, name)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); - } - value = Py_NewRef(Py_None); - } - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - } - PyErr_SetRaisedException(exc); + return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { - _PyFrame_LocalsToFast(f->f_frame, clear); - f->f_fast_as_locals = 0; - } + return; } int diff --git a/Python/ceval.c b/Python/ceval.c index 4b1563f353271d..3090332073a384 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2465,7 +2465,7 @@ _PyEval_GetFrameLocals(void) return NULL; } - locals = _PyFrame_GetLocals(current_frame, 1); + locals = _PyFrame_GetLocals(current_frame); if (locals == NULL) { return NULL; } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index d3146973b75178..d581734e878fc5 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -127,7 +127,7 @@ import_star(PyThreadState* tstate, PyObject *from) return NULL; } - PyObject *locals = frame->f_locals; + PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); @@ -135,6 +135,7 @@ import_star(PyThreadState* tstate, PyObject *from) } int err = import_all_from(tstate, locals, from); _PyFrame_LocalsToFast(frame, 0); + Py_DECREF(locals); if (err < 0) { return NULL; } From ff886ffe922d4ae29aeaca6e38ad806fede1cc9e Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 1 Mar 2024 18:34:17 -0800 Subject: [PATCH 08/43] Update tests for the new f_locals --- Lib/test/test_listcomps.py | 4 ++-- Lib/test/test_sys.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index f95a78aff0c711..0943afb242456b 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -610,10 +610,10 @@ def test_exception_in_post_comp_call(self): def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys - self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) def _recursive_replace(self, maybe_code): if not isinstance(maybe_code, types.CodeType): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 71671a5a984256..c9e1715bba598d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1550,7 +1550,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pi3cP7P2ic??2P')) # function def func(): pass check(func, size('15Pi')) From 9690a2d672572de6c3ba745c814690795e21d791 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 1 Mar 2024 18:50:14 -0800 Subject: [PATCH 09/43] Comment out the pop for now --- Lib/test/test_frame.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index baed03d92b9e56..713aec0bae6225 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -201,7 +201,8 @@ def inner(): def test_locals(self): f, outer, inner = self.make_frames() outer_locals = outer.f_locals - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) + # TODO: Support pop for f_locals + # self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) self.assertEqual(outer_locals, {'x': 5, 'y': 6}) inner_locals = inner.f_locals self.assertEqual(inner_locals, {'x': 5, 'z': 7}) From 6e9848af519d20703c38bbf1a16872686ab2a6d8 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 1 Mar 2024 19:00:13 -0800 Subject: [PATCH 10/43] Convert f_locals to dict first --- Lib/test/test_frame.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 713aec0bae6225..bd2819c3f2babf 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -200,10 +200,10 @@ def inner(): def test_locals(self): f, outer, inner = self.make_frames() - outer_locals = outer.f_locals # TODO: Support pop for f_locals - # self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(outer_locals, {'x': 5, 'y': 6}) + outer_locals = dict(outer.f_locals) + self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) + self.assertEqual(dict(outer_locals), {'x': 5, 'y': 6}) inner_locals = inner.f_locals self.assertEqual(inner_locals, {'x': 5, 'z': 7}) From b84b0df4f5722476d6c870425e02fb4a22b99150 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 24 Apr 2024 14:27:29 -0700 Subject: [PATCH 11/43] Add static to static functions, add interface for new C API --- Include/ceval.h | 4 ++++ Objects/frameobject.c | 22 +++++++++++----------- Python/ceval.c | 22 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index 9885bdb7febc21..2902ee1b34937d 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -22,6 +22,10 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void); + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ca811696abef4e..a25a85d25a56bd 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -19,7 +19,7 @@ // Utilities // Returns borrowed reference or NULL -PyObject * +static PyObject * framelocalproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) { PyObject **fast = _PyFrame_GetLocalsArray(frame->f_frame); @@ -52,7 +52,7 @@ framelocalproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) // Type functions -PyObject * +static PyObject * framelocalsproxy_keys(PyObject *self, PyObject *__unused) { PyObject *names = PyList_New(0); @@ -82,7 +82,7 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) return names; } -void +static void framelocalsproxy_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); @@ -90,7 +90,7 @@ framelocalsproxy_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } -PyObject * +static PyObject * framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); @@ -122,20 +122,20 @@ framelocalsproxy_tp_clear(PyObject *self) return 0; } -int +static int framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) { Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); return 0; } -PyObject * +static PyObject * framelocalsproxy_iter(PyObject *self) { return PyObject_GetIter(framelocalsproxy_keys(self, NULL)); } -PyObject * +static PyObject * framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) { if (PyFrameLocalsProxy_Check(other)) { @@ -159,7 +159,7 @@ framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) // Methods -PyObject * +static PyObject * framelocalsproxy_items(PyObject *self, PyObject *__unused) { PyObject *items = PyList_New(0); @@ -191,7 +191,7 @@ framelocalsproxy_items(PyObject *self, PyObject *__unused) return items; } -Py_ssize_t +static Py_ssize_t framelocalsproxy_length(PyObject *self) { PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; @@ -211,7 +211,7 @@ framelocalsproxy_length(PyObject *self) return size; } -PyObject * +static PyObject * framelocalsproxy_getitem(PyObject *self, PyObject *key) { PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; @@ -242,7 +242,7 @@ framelocalsproxy_getitem(PyObject *self, PyObject *key) return NULL; } -int +static int framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) { /* Merge locals into fast locals */ diff --git a/Python/ceval.c b/Python/ceval.c index 3090332073a384..fbe88dc4b0e59a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2518,6 +2518,28 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject* +PyEval_GetFrameLocals(void) +{ + return _PyEval_GetFrameLocals(); +} + +PyObject* PyEval_GetFrameGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + return NULL; + } + return Py_XNewRef(current_frame->f_globals); +} + +PyObject* PyEval_GetFrameBuiltins(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return Py_XNewRef(_PyEval_GetBuiltins(tstate)); +} + int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { From bebff2889a8252117559f350d894db7227674a3f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 26 Apr 2024 11:25:13 -0700 Subject: [PATCH 12/43] Add some tests and a few methods --- Lib/test/test_frame.py | 31 +++++++++++++++++++++++++++++++ Objects/frameobject.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index bd2819c3f2babf..3c4fef50127f1b 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -270,6 +270,37 @@ def inner(): r"^$" % (file_repr, offset + 5)) +class TestFrameLocals(unittest.TestCase): + def test_scope(self): + class A: + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + + self.assertEqual(A.x, 2) + self.assertEqual(A.y, 2) + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + self.assertEqual(x, 2) + self.assertEqual(locals()['y'], 2) + f() + + def test_as_dict(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + # self, x, y, d + self.assertEqual(len(d), 4) + self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) + self.assertEqual(len(d.values()), 4) + self.assertIn(1, d.values()) + self.assertEqual(len(d.items()), 4) + self.assertIn(('x', 1), d.items()) + self.assertEqual(d.__getitem__('x'), 1) + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a25a85d25a56bd..15fa9cad390db9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -159,6 +159,32 @@ framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) // Methods +static PyObject* +framelocalsproxy_values(PyObject *self, PyObject *__unused) +{ + PyObject *values = PyList_New(0); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *value = framelocalproxy_getval(frame, co, i); + if (value) { + PyList_Append(values, value); + } + } + + // Iterate through the extra locals + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyList_Append(values, value); + } + + return values; +} + static PyObject * framelocalsproxy_items(PyObject *self, PyObject *__unused) { @@ -320,8 +346,12 @@ static PyMappingMethods framelocalsproxy_as_mapping = { static PyMethodDef framelocalsproxy_methods[] = { {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, NULL}, + {"__setitem__", framelocalsproxy_setitem, METH_O | METH_COEXIST, + NULL}, {"keys", framelocalsproxy_keys, METH_NOARGS, NULL}, + {"values", framelocalsproxy_values, METH_NOARGS, + NULL}, {"items", framelocalsproxy_items, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ From d846de9a1cdb6eb6a91496562adae04173c4d13c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 13:27:11 -0700 Subject: [PATCH 13/43] Implement all methods --- Lib/test/test_frame.py | 72 ++++++++++ Lib/test/test_peepholer.py | 17 --- Misc/stable_abi.toml | 6 + Objects/frameobject.c | 288 ++++++++++++++++++++++++++++++++++--- Python/sysmodule.c | 3 + 5 files changed, 351 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 3c4fef50127f1b..4014b2bf82091c 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,3 +1,4 @@ +import copy import gc import operator import re @@ -300,6 +301,77 @@ def test_as_dict(self): self.assertEqual(len(d.items()), 4) self.assertIn(('x', 1), d.items()) self.assertEqual(d.__getitem__('x'), 1) + d.__setitem__('x', 2) + self.assertEqual(d['x'], 2) + self.assertEqual(d.get('x'), 2) + self.assertIs(d.get('non_exist', None), None) + self.assertEqual(d.__len__(), 4) + self.assertEqual(set([key for key in d]), set(['x', 'y', 'd', 'self'])) + self.assertIn('x', d) + self.assertTrue(d.__contains__('x')) + + self.assertEqual(reversed(d), list(reversed(d.keys()))) + + d.update({'x': 3, 'z': 4}) + self.assertEqual(d['x'], 3) + self.assertEqual(d['z'], 4) + + with self.assertRaises(TypeError): + d.update([1, 2]) + + self.assertEqual(d.setdefault('x', 5), 3) + self.assertEqual(d.setdefault('new', 5), 5) + self.assertEqual(d['new'], 5) + + def test_as_number(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + self.assertIn('z', d | {'z': 3}) + d |= {'z': 3} + self.assertEqual(d['z'], 3) + d |= {'y': 3} + self.assertEqual(d['y'], 3) + with self.assertRaises(TypeError): + d |= 3 + with self.assertRaises(TypeError): + _ = d | [3] + + def test_repr(self): + x = 1 + # Introduce a reference cycle + frame = sys._getframe() + self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) + + def test_delete(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + del d['x'] + + with self.assertRaises(TypeError): + d.clear() + + with self.assertRaises(TypeError): + d.pop('x') + + def test_unsupport(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + d.copy() + + with self.assertRaises(TypeError): + copy.copy(d) + + with self.assertRaises(TypeError): + copy.deepcopy(d) + + with self.assertRaises(TypeError): + d.get(1) + + with self.assertRaises(TypeError): + d.setdefault(1, 'x') class TestIncompleteFrameAreInvisible(unittest.TestCase): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 2ea186c85c8823..1b34d0bd449ee4 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -932,23 +932,6 @@ def f(): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") return f - def test_deleting_local_warns_and_assigns_none(self): - f = self.make_function_with_no_checks() - co_code = f.__code__.co_code - def trace(frame, event, arg): - if event == 'line' and frame.f_lineno == 4: - del frame.f_locals["x"] - sys.settrace(None) - return None - return trace - e = r"assigning None to unbound local 'x'" - with self.assertWarnsRegex(RuntimeWarning, e): - sys.settrace(trace) - f() - self.assertInBytecode(f, "LOAD_FAST") - self.assertNotInBytecode(f, "LOAD_FAST_CHECK") - self.assertEqual(f.__code__.co_code, co_code) - def test_modifying_local_does_not_add_check(self): f = self.make_function_with_no_checks() def trace(frame, event, arg): diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index a9875f6ffd1a56..eafa171b1ad757 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2489,3 +2489,9 @@ added = '3.13' [function.PyList_GetItemRef] added = '3.13' +[function.PyEval_GetFrameBuiltins] + added = '3.13 +[function.PyEval_GetFrameGlobals] + added = '3.13 +[function.PyEval_GetFrameLocals] + added = '3.13 diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 15fa9cad390db9..bcfea9adc71319 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -20,7 +20,7 @@ // Returns borrowed reference or NULL static PyObject * -framelocalproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) +framelocalsproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) { PyObject **fast = _PyFrame_GetLocalsArray(frame->f_frame); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -50,6 +50,49 @@ framelocalproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) return value; } +static int +framelocalsproxy_merge(PyObject* self, PyObject* other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + return -1; + } + + PyObject *keys = PyMapping_Keys(other); + PyObject *iter = NULL; + PyObject *key = NULL; + PyObject *value = NULL; + + assert(keys != NULL); + + iter = PyObject_GetIter(keys); + Py_DECREF(keys); + + if (iter == NULL) { + return -1; + } + + while ((key = PyIter_Next(iter)) != NULL) { + value = PyObject_GetItem(other, key); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(iter); + return -1; + } + + if (PyObject_SetItem(self, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); + return -1; + } + + Py_DECREF(key); + Py_DECREF(value); + } + + return 0; +} + // Type functions static PyObject * @@ -60,7 +103,7 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) PyCodeObject *co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *val = framelocalproxy_getval(frame, co, i); + PyObject *val = framelocalsproxy_getval(frame, co, i); if (val) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyList_Append(names, name); @@ -157,6 +200,61 @@ framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_NOTIMPLEMENTED; } +static PyObject * +framelocalsproxy_repr(PyObject *self) +{ + int i = Py_ReprEnter(self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("{...}") : NULL; + } + + PyObject *dct = PyDict_New(); + PyObject *repr = NULL; + + if (PyDict_Update(dct, self) == 0) { + repr = PyObject_Repr(dct); + } + Py_ReprLeave(self); + + Py_DECREF(dct); + return repr; +} + +static PyObject* +framelocalsproxy_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + PyObject *result = PyDict_New(); + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + if (PyDict_Update(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_inplace_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (framelocalsproxy_merge(self, other) < 0) { + Py_RETURN_NOTIMPLEMENTED; + } + + return Py_NewRef(self); +} + // Methods static PyObject* @@ -167,7 +265,7 @@ framelocalsproxy_values(PyObject *self, PyObject *__unused) PyCodeObject *co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value = framelocalproxy_getval(frame, co, i); + PyObject *value = framelocalsproxy_getval(frame, co, i); if (value) { PyList_Append(values, value); } @@ -194,7 +292,7 @@ framelocalsproxy_items(PyObject *self, PyObject *__unused) for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = framelocalproxy_getval(frame, co, i); + PyObject *value = framelocalsproxy_getval(frame, co, i); if (value) { PyObject *pair = PyTuple_Pack(2, name, value); @@ -230,7 +328,7 @@ framelocalsproxy_length(PyObject *self) } for (int i = 0; i < co->co_nlocalsplus; i++) { - if (framelocalproxy_getval(frame, co, i) != NULL) { + if (framelocalsproxy_getval(frame, co, i) != NULL) { size++; } } @@ -247,7 +345,7 @@ framelocalsproxy_getitem(PyObject *self, PyObject *key) // The name and the key might be interned so do a fast check first if (PyTuple_GET_ITEM(co->co_localsplusnames, i) == key || _PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { - PyObject *value = framelocalproxy_getval(frame, co, i); + PyObject *value = framelocalsproxy_getval(frame, co, i); if (value != NULL) { return Py_NewRef(value); } @@ -276,6 +374,11 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); PyCodeObject* co = PyFrame_GetCode(frame); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return -1; + } + for (int i = 0; i < co->co_nlocalsplus; i++) { if (PyTuple_GET_ITEM(co->co_localsplusnames, i) == key || _PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { @@ -300,16 +403,6 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) Py_XDECREF(oldvalue); } } else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, key)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame); - } - value = Py_NewRef(Py_None); - } Py_XSETREF(fast[i], Py_NewRef(value)); } Py_XDECREF(value); @@ -337,6 +430,145 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return 0; } +static PyObject* +framelocalsproxy_contains(PyObject *self, PyObject *key) +{ + PyObject* keys = framelocalsproxy_keys(self, NULL); + + if (keys == NULL) { + return NULL; + } + + int result = PySequence_Contains(keys, key); + + Py_DECREF(keys); + + if (result < 0) { + return NULL; + } + + return PyBool_FromLong(result); +} + +static PyObject* +framelocalsproxy_update(PyObject *self, PyObject *other) +{ + if (framelocalsproxy_merge(self, other) < 0) { + PyErr_SetString(PyExc_TypeError, "update() argument must be dict or another FrameLocalsProxy"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "get expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + PyObject *result = NULL; + + if (nargs == 2) { + default_value = args[1]; + } + + if (!PyUnicode_CheckExact(key)) { + PyErr_SetString(PyExc_TypeError, "key must be a string"); + return NULL; + } + + result = framelocalsproxy_getitem(self, args[0]); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "setdefault expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + PyObject *result = NULL; + + if (nargs == 2) { + default_value = args[1]; + } + + if (!PyUnicode_CheckExact(key)) { + PyErr_SetString(PyExc_TypeError, "key must be a string"); + return NULL; + } + + result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + if (framelocalsproxy_setitem(self, key, default_value) < 0) { + return NULL; + } + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_reversed(PyObject *self, PyObject *__unused) +{ + PyObject *result = framelocalsproxy_keys(self, NULL); + if (PyList_Reverse(result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + +static PyObject* +framelocalsproxy_copy_unsupported(PyObject *self, PyObject *__unused) +{ + PyErr_SetString(PyExc_TypeError, "FrameLocalsProxy is uncopyable"); + return NULL; +} + +static PyObject* +framelocalsproxy_clear_unsupported(PyObject *self, PyObject *__unused) +{ + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return NULL; +} + +static PyObject* +framelocalsproxy_pop_unsupported(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return NULL; +} + +static PyNumberMethods framelocalsproxy_as_number = { + .nb_or = framelocalsproxy_or, + .nb_inplace_or = framelocalsproxy_inplace_or, +}; + static PyMappingMethods framelocalsproxy_as_mapping = { framelocalsproxy_length, // mp_length framelocalsproxy_getitem, // mp_subscript @@ -344,9 +576,19 @@ static PyMappingMethods framelocalsproxy_as_mapping = { }; static PyMethodDef framelocalsproxy_methods[] = { + {"__contains__", framelocalsproxy_contains, METH_O | METH_COEXIST, + NULL}, {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, NULL}, - {"__setitem__", framelocalsproxy_setitem, METH_O | METH_COEXIST, + {"__reversed__", framelocalsproxy_reversed, METH_NOARGS, + NULL}, + {"__copy__", framelocalsproxy_copy_unsupported, METH_NOARGS, + NULL}, + {"__deepcopy__", framelocalsproxy_copy_unsupported, METH_NOARGS, + NULL}, + {"copy", framelocalsproxy_copy_unsupported, METH_NOARGS, + NULL}, + {"clear", framelocalsproxy_clear_unsupported, METH_NOARGS, NULL}, {"keys", framelocalsproxy_keys, METH_NOARGS, NULL}, @@ -354,6 +596,14 @@ static PyMethodDef framelocalsproxy_methods[] = { NULL}, {"items", framelocalsproxy_items, METH_NOARGS, NULL}, + {"update", framelocalsproxy_update, METH_O, + NULL}, + {"pop", _PyCFunction_CAST(framelocalsproxy_pop_unsupported), METH_FASTCALL, + NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + NULL}, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + NULL}, {NULL, NULL} /* sentinel */ }; @@ -362,6 +612,8 @@ PyTypeObject PyFrameLocalsProxy_Type = { .tp_name = "FrameLocalsProxy", .tp_basicsize = sizeof(PyFrameLocalsProxyObject), .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_repr = &framelocalsproxy_repr, + .tp_as_number = &framelocalsproxy_as_number, .tp_as_mapping = &framelocalsproxy_as_mapping, .tp_getattro = PyObject_GenericGetAttr, .tp_setattro = PyObject_GenericSetAttr, @@ -1597,7 +1849,7 @@ _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) } PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject* value = framelocalproxy_getval(f, co, i); + PyObject* value = framelocalsproxy_getval(f, co, i); if (value == NULL) { continue; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 437d7f8dfc4958..8fccfef35fad5a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1861,6 +1861,7 @@ _PySys_GetSizeOf(PyObject *o) } else { res = _PyObject_CallNoArgs(method); + // printf("size: %s\n", PyUnicode_AsUTF8(PyObject_Repr(res))); Py_DECREF(method); } @@ -1868,6 +1869,7 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; size = PyLong_AsSsize_t(res); + printf("size: %ld\n", size); Py_DECREF(res); if (size == -1 && _PyErr_Occurred(tstate)) return (size_t)-1; @@ -1885,6 +1887,7 @@ _PySys_GetSizeOf(PyObject *o) /* Add the size of the pre-header if "o" is not a static type */ presize = _PyType_PreHeaderSize(Py_TYPE(o)); } + printf("presize: %ld\n", presize); return (size_t)size + presize; } From d00a74295b744dcf55baa5a3e28a78acca4cf2c7 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 13:31:13 -0700 Subject: [PATCH 14/43] Make f_extra_locals extra lazy --- Objects/frameobject.c | 46 ++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index bcfea9adc71319..9ba34a11b6f4c8 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -146,15 +146,6 @@ framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); - if (frame->f_extra_locals == NULL) { - frame->f_extra_locals = PyDict_New(); - if (frame->f_extra_locals == NULL) { - Py_DECREF(frame); - Py_DECREF(self); - return NULL; - } - } - return (PyObject *)self; } @@ -276,8 +267,10 @@ framelocalsproxy_values(PyObject *self, PyObject *__unused) PyObject *key = NULL; PyObject *value = NULL; - while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { - PyList_Append(values, value); + if (frame->f_extra_locals) { + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyList_Append(values, value); + } } return values; @@ -306,10 +299,12 @@ framelocalsproxy_items(PyObject *self, PyObject *__unused) PyObject *key = NULL; PyObject *value = NULL; - while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { - PyObject *pair = PyTuple_Pack(2, key, value); - PyList_Append(items, pair); - Py_DECREF(pair); + if (frame->f_extra_locals) { + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject *pair = PyTuple_Pack(2, key, value); + PyList_Append(items, pair); + Py_DECREF(pair); + } } return items; @@ -414,17 +409,18 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) PyObject *extra = frame->f_extra_locals; - if (extra != NULL) { - assert(PyDict_Check(extra)); - if (value == NULL) { - if (PyDict_DelItem(extra, key) < 0) { - return -1; - } - } else { - if (PyDict_SetItem(extra, key, value) < 0) { - return -1; - } + if (extra == NULL) { + extra = PyDict_New(); + if (extra == NULL) { + return -1; } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + if (PyDict_SetItem(extra, key, value) < 0) { + return -1; } return 0; From f720e1287970c168ee259c4b47b81aaeddd04956 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 13:33:37 -0700 Subject: [PATCH 15/43] Fix typo --- Misc/stable_abi.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index e8545fb970de16..77473662aaa76c 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2502,8 +2502,8 @@ [function.PyType_GetModuleByDef] added = '3.13' [function.PyEval_GetFrameBuiltins] - added = '3.13 + added = '3.13' [function.PyEval_GetFrameGlobals] - added = '3.13 + added = '3.13' [function.PyEval_GetFrameLocals] - added = '3.13 + added = '3.13' From 64d3772981ffbb35cf78bac587461199acb5502c Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 13:34:35 -0700 Subject: [PATCH 16/43] Remove print debugging --- Python/sysmodule.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index df0693ca34b930..7af363678e8e86 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1889,7 +1889,6 @@ _PySys_GetSizeOf(PyObject *o) } else { res = _PyObject_CallNoArgs(method); - // printf("size: %s\n", PyUnicode_AsUTF8(PyObject_Repr(res))); Py_DECREF(method); } @@ -1897,7 +1896,6 @@ _PySys_GetSizeOf(PyObject *o) return (size_t)-1; size = PyLong_AsSsize_t(res); - printf("size: %ld\n", size); Py_DECREF(res); if (size == -1 && _PyErr_Occurred(tstate)) return (size_t)-1; @@ -1915,7 +1913,6 @@ _PySys_GetSizeOf(PyObject *o) /* Add the size of the pre-header if "o" is not a static type */ presize = _PyType_PreHeaderSize(Py_TYPE(o)); } - printf("presize: %ld\n", presize); return (size_t)size + presize; } From 2eadbf0d7a42fa2becd2743ab52c788080bf7afd Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 13:36:51 -0700 Subject: [PATCH 17/43] Fix some styling issue --- Objects/frameobject.c | 6 ++---- Python/ceval.c | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 025d91b72acee5..f212133d5d4aca 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -467,7 +467,6 @@ framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) PyObject *key = args[0]; PyObject *default_value = Py_None; - PyObject *result = NULL; if (nargs == 2) { default_value = args[1]; @@ -478,7 +477,7 @@ framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) return NULL; } - result = framelocalsproxy_getitem(self, args[0]); + PyObject *result = framelocalsproxy_getitem(self, args[0]); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) { @@ -501,7 +500,6 @@ framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t na PyObject *key = args[0]; PyObject *default_value = Py_None; - PyObject *result = NULL; if (nargs == 2) { default_value = args[1]; @@ -512,7 +510,7 @@ framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t na return NULL; } - result = framelocalsproxy_getitem(self, key); + PyObject *result = framelocalsproxy_getitem(self, key); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) { diff --git a/Python/ceval.c b/Python/ceval.c index 74d72f5466aae6..cce5a45a04852c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2481,14 +2481,13 @@ PyObject * _PyEval_GetFrameLocals(void) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject* locals = NULL; _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); return NULL; } - locals = _PyFrame_GetLocals(current_frame); + PyObject *locals = _PyFrame_GetLocals(current_frame); if (locals == NULL) { return NULL; } From cbae1995cc84fbd81764d13300c98c3a53ee7631 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 14:02:18 -0700 Subject: [PATCH 18/43] Update generated files for cAPI --- Doc/data/stable_abi.dat | 3 +++ Lib/test/test_stable_abi_ctypes.py | 3 +++ PC/python3dll.c | 3 +++ 3 files changed, 9 insertions(+) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 8c8a378f52bd5d..76a035f194d911 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -188,6 +188,9 @@ function,PyEval_EvalFrame,3.2,, function,PyEval_EvalFrameEx,3.2,, function,PyEval_GetBuiltins,3.2,, function,PyEval_GetFrame,3.2,, +function,PyEval_GetFrameBuiltins,3.13,, +function,PyEval_GetFrameGlobals,3.13,, +function,PyEval_GetFrameLocals,3.13,, function,PyEval_GetFuncDesc,3.2,, function,PyEval_GetFuncName,3.2,, function,PyEval_GetGlobals,3.2,, diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index d22698168615e2..c06c285c5013a6 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -227,6 +227,9 @@ def test_windows_feature_macros(self): "PyEval_EvalFrameEx", "PyEval_GetBuiltins", "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", "PyEval_GetFuncDesc", "PyEval_GetFuncName", "PyEval_GetGlobals", diff --git a/PC/python3dll.c b/PC/python3dll.c index c6fdc0bd73b9fe..86c888430891c9 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -253,6 +253,9 @@ EXPORT_FUNC(PyEval_EvalFrame) EXPORT_FUNC(PyEval_EvalFrameEx) EXPORT_FUNC(PyEval_GetBuiltins) EXPORT_FUNC(PyEval_GetFrame) +EXPORT_FUNC(PyEval_GetFrameBuiltins) +EXPORT_FUNC(PyEval_GetFrameGlobals) +EXPORT_FUNC(PyEval_GetFrameLocals) EXPORT_FUNC(PyEval_GetFuncDesc) EXPORT_FUNC(PyEval_GetFuncName) EXPORT_FUNC(PyEval_GetGlobals) From 9e7edf8b9abe8396655078def8372561165a5a3f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 14:42:08 -0700 Subject: [PATCH 19/43] Remove f_fast_as_locals and useless calls for sys.settrace --- Include/internal/pycore_frame.h | 1 - Lib/test/test_sys.py | 2 +- Objects/frameobject.c | 1 - Python/intrinsics.c | 1 - Python/sysmodule.c | 8 -------- 5 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 5695d955e04d81..f84fc5eab59dbf 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -25,7 +25,6 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index d3979dd6311a75..a00a3ee57a89f7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1555,7 +1555,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3cP7P2ic??2P')) + check(x, size('3Pi2cP7P2ic??2P')) # function def func(): pass check(func, size('15Pi')) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f212133d5d4aca..08f3491ddca8e0 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1655,7 +1655,6 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace = NULL; f->f_trace_lines = 1; f->f_trace_opcodes = 0; - f->f_fast_as_locals = 0; f->f_lineno = 0; f->f_extra_locals = NULL; return f; diff --git a/Python/intrinsics.c b/Python/intrinsics.c index d581734e878fc5..430214e196c901 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -134,7 +134,6 @@ import_star(PyThreadState* tstate, PyObject *from) return NULL; } int err = import_all_from(tstate, locals, from); - _PyFrame_LocalsToFast(frame, 0); Py_DECREF(locals); if (err < 0) { return NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7af363678e8e86..809b30fa0d3231 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1022,13 +1022,6 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - /* Discard any previous modifications the frame's fast locals */ - if (frame->f_fast_as_locals) { - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - } - /* call the Python-level function */ if (arg == NULL) { arg = Py_None; @@ -1036,7 +1029,6 @@ call_trampoline(PyThreadState *tstate, PyObject* callback, PyObject *args[3] = {(PyObject *)frame, whatstrings[what], arg}; PyObject *result = _PyObject_VectorcallTstate(tstate, callback, args, 3, NULL); - PyFrame_LocalsToFast(frame, 1); return result; } From 523cb756f329551e4396ecd0f4b2953d0964d5e2 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 21:44:42 +0000 Subject: [PATCH 20/43] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst new file mode 100644 index 00000000000000..f6293c287b7ade --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst @@ -0,0 +1 @@ +Implement PEP 667 - converted ``frame.f_locals`` to a real-time proxy From 4b83311ae1f3fa07b962d02b100ba3a6a867c686 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 15:02:02 -0700 Subject: [PATCH 21/43] Add the new type to static types --- Objects/object.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/object.c b/Objects/object.c index 91bb0114cbfc32..0d8dc9ea498b19 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2235,6 +2235,7 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, From ae2db7cea037e476c1bf354903f7d08002612f06 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 15:16:20 -0700 Subject: [PATCH 22/43] Remove internal APIs for fast locals --- Include/internal/pycore_frame.h | 6 ------ Objects/frameobject.c | 13 ------------- Python/ceval.c | 7 +------ Python/intrinsics.c | 3 --- 4 files changed, 1 insertion(+), 28 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index f84fc5eab59dbf..af8a00e82a6906 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -248,12 +248,6 @@ _PyFrame_GetLocals(_PyInterpreterFrame *frame); PyObject * _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame); -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear); - static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 08f3491ddca8e0..c9ed25d9afdce9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1853,13 +1853,6 @@ _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) -{ - return 0; -} - - PyObject * PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name) { @@ -1919,12 +1912,6 @@ PyFrame_FastToLocals(PyFrameObject *f) return; } -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) -{ - return; -} - void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { diff --git a/Python/ceval.c b/Python/ceval.c index cce5a45a04852c..2c23c56771c96a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2468,12 +2468,7 @@ PyEval_GetLocals(void) return NULL; } - if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { - return NULL; - } - - PyObject *locals = current_frame->f_locals; - assert(locals != NULL); + PyObject *locals = _PyEval_GetFrameLocals(); return locals; } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 430214e196c901..5e03c5b2a27392 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -123,9 +123,6 @@ static PyObject * import_star(PyThreadState* tstate, PyObject *from) { _PyInterpreterFrame *frame = tstate->current_frame; - if (_PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { From bf45c02965b2fc53ca653e12f3e6924cb8b2ab4b Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 15:20:38 -0700 Subject: [PATCH 23/43] Add extra tests for closure --- Lib/test/test_frame.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 35cbb7105f58c9..1523b8bd60acbf 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -289,6 +289,22 @@ def f(): self.assertEqual(locals()['y'], 2) f() + def test_closure(self): + x = 1 + y = 2 + + def f(): + z = x + y + d = sys._getframe().f_locals + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 2) + d['x'] = 2 + d['y'] = 3 + + f() + self.assertEqual(x, 2) + self.assertEqual(y, 3) + def test_as_dict(self): x = 1 y = 2 From 026e15ea1334e3e7d91f45e1baac94fa8e47e3fe Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 15:36:47 -0700 Subject: [PATCH 24/43] Add the type to globals-to-fix --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 79a8f850ec1451..8bb8a7363e99fa 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -43,6 +43,7 @@ Objects/enumobject.c - PyReversed_Type - Objects/fileobject.c - PyStdPrinter_Type - Objects/floatobject.c - PyFloat_Type - Objects/frameobject.c - PyFrame_Type - +Objects/frameobject.c - PyFrameLocalsProxy_Type - Objects/funcobject.c - PyClassMethod_Type - Objects/funcobject.c - PyFunction_Type - Objects/funcobject.c - PyStaticMethod_Type - From f42980d2af923492e2d5a40273d6d8b011e3612a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 16:28:32 -0700 Subject: [PATCH 25/43] Add CAapi test --- Lib/test/test_frame.py | 53 ++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 1523b8bd60acbf..90362c7a201b9a 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -14,7 +14,7 @@ _testcapi = None from test import support -from test.support import threading_helper, Py_GIL_DISABLED +from test.support import import_helper, threading_helper, Py_GIL_DISABLED from test.support.script_helper import assert_python_ok @@ -199,15 +199,6 @@ def inner(): tb = tb.tb_next return frames - def test_locals(self): - f, outer, inner = self.make_frames() - # TODO: Support pop for f_locals - outer_locals = dict(outer.f_locals) - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(dict(outer_locals), {'x': 5, 'y': 6}) - inner_locals = inner.f_locals - self.assertEqual(inner_locals, {'x': 5, 'z': 7}) - def test_clear_locals(self): # Test f_locals after clear() (issue #21897) f, outer, inner = self.make_frames() @@ -219,8 +210,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -371,6 +362,11 @@ def test_delete(self): with self.assertRaises(TypeError): d.pop('x') + @support.cpython_only + def test_sizeof(self): + proxy = sys._getframe().f_locals + support.check_sizeof(self, proxy, support.calcobjsize("P")) + def test_unsupport(self): x = 1 d = sys._getframe().f_locals @@ -389,6 +385,39 @@ def test_unsupport(self): with self.assertRaises(TypeError): d.setdefault(1, 'x') + +class TestFrameCApi(unittest.TestCase): + def test_basic(self): + x = 1 + ctypes = import_helper.import_module('ctypes') + PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals + PyEval_GetFrameLocals.restype = ctypes.py_object + frame_locals = PyEval_GetFrameLocals() + self.assertTrue(type(frame_locals), dict) + self.assertEqual(frame_locals['x'], 1) + frame_locals['x'] = 2 + self.assertEqual(x, 1) + + PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals + PyEval_GetFrameGlobals.restype = ctypes.py_object + frame_globals = PyEval_GetFrameGlobals() + self.assertTrue(type(frame_globals), dict) + + PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins + PyEval_GetFrameBuiltins.restype = ctypes.py_object + frame_builtins = PyEval_GetFrameBuiltins() + self.assertEqual(frame_builtins, __builtins__) + + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocals.argtypes = [ctypes.py_object] + PyFrame_GetLocals.restype = ctypes.py_object + frame = sys._getframe() + f_locals = PyFrame_GetLocals(frame) + self.assertTrue(f_locals['x'], 1) + f_locals['x'] = 2 + self.assertEqual(x, 2) + + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): From e693ad04004be1abaed8fbfbb41c29ed6e55c191 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 27 Apr 2024 16:28:45 -0700 Subject: [PATCH 26/43] Polish lint --- Objects/frameobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c9ed25d9afdce9..6b6f5e7018b217 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -13,6 +13,7 @@ #include "pycore_frame.h" #include "opcode.h" // EXTENDED_ARG + #define OFF(x) offsetof(PyFrameObject, x) From 5dd045ba2dce921e02a024d3f1e62e152da1bbbe Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 28 Apr 2024 11:08:53 -0700 Subject: [PATCH 27/43] Apply some simple changes --- Lib/test/test_frame.py | 1 + Objects/frameobject.c | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 90362c7a201b9a..0206a47b8f04b5 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -302,6 +302,7 @@ def test_as_dict(self): d = sys._getframe().f_locals # self, x, y, d self.assertEqual(len(d), 4) + self.assertIs(d['d'], d) self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) self.assertEqual(len(d.values()), 4) self.assertIn(1, d.values()) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6b6f5e7018b217..3b00c4f311e111 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -420,11 +420,7 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) assert(PyDict_Check(extra)); - if (PyDict_SetItem(extra, key, value) < 0) { - return -1; - } - - return 0; + return PyDict_SetItem(extra, key, value) < 0; } static PyObject* From 30ecd4deb9fa58812ffbb6e1eeee7a17e635a766 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 28 Apr 2024 11:10:00 -0700 Subject: [PATCH 28/43] Update Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst Co-authored-by: Mark Shannon --- .../2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst index f6293c287b7ade..46e628f7fa7dbe 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst @@ -1 +1 @@ -Implement PEP 667 - converted ``frame.f_locals`` to a real-time proxy +Implement PEP 667 - converted ``frame.f_locals`` to a write through proxy From f35c5e3b9cd68c09af99997d7b13b36126d1eac7 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 28 Apr 2024 11:27:17 -0700 Subject: [PATCH 29/43] Abstract the key index part --- Objects/frameobject.c | 112 +++++++++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3b00c4f311e111..ea6735c130176a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -19,6 +19,29 @@ // Utilities +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key) +{ + PyCodeObject *co = PyFrame_GetCode(frame); + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + return i; + } + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (_PyUnicode_EQ(name, key)) { + return i; + } + } + + return -1; +} + // Returns borrowed reference or NULL static PyObject * framelocalsproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) @@ -337,14 +360,11 @@ framelocalsproxy_getitem(PyObject *self, PyObject *key) PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject* co = PyFrame_GetCode(frame); - for (int i = 0; i < co->co_nlocalsplus; i++) { - // The name and the key might be interned so do a fast check first - if (PyTuple_GET_ITEM(co->co_localsplusnames, i) == key || - _PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { - PyObject *value = framelocalsproxy_getval(frame, co, i); - if (value != NULL) { - return Py_NewRef(value); - } + int i = framelocalsproxy_getkeyindex(frame, key); + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame, co, i); + if (value != NULL) { + return Py_NewRef(value); } } @@ -375,35 +395,34 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - for (int i = 0; i < co->co_nlocalsplus; i++) { - if (PyTuple_GET_ITEM(co->co_localsplusnames, i) == key || - _PyUnicode_EQ(PyTuple_GET_ITEM(co->co_localsplusnames, i), key)) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); + int i = framelocalsproxy_getkeyindex(frame, key); + + if (i >= 0) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue != NULL && PyCell_Check(oldvalue)); + cell = oldvalue; + } else if (kind & CO_FAST_CELL && oldvalue != NULL) { + if (PyCell_Check(oldvalue)) { cell = oldvalue; - } else if (kind & CO_FAST_CELL && oldvalue != NULL) { - if (PyCell_Check(oldvalue)) { - cell = oldvalue; - } } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } else if (value != oldvalue) { - Py_XSETREF(fast[i], Py_NewRef(value)); + } + if (cell != NULL) { + oldvalue = PyCell_GET(cell); + if (value != oldvalue) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue); } - Py_XDECREF(value); - return 0; + } else if (value != oldvalue) { + Py_XSETREF(fast[i], Py_NewRef(value)); } + Py_XDECREF(value); + return 0; } // Okay not in the fast locals, try extra locals @@ -426,21 +445,24 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) static PyObject* framelocalsproxy_contains(PyObject *self, PyObject *key) { - PyObject* keys = framelocalsproxy_keys(self, NULL); - - if (keys == NULL) { - return NULL; - } - - int result = PySequence_Contains(keys, key); - - Py_DECREF(keys); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + int i = framelocalsproxy_getkeyindex(frame, key); - if (result < 0) { - return NULL; + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame, PyFrame_GetCode(frame), i); + if (value != NULL) { + Py_RETURN_TRUE; + } + } else { + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + if (PyDict_Contains(extra, key)) { + Py_RETURN_TRUE; + } + } } - return PyBool_FromLong(result); + Py_RETURN_FALSE; } static PyObject* From 5844fb4eaec9fab58edc87ec12914a6057ebae7a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 28 Apr 2024 11:29:54 -0700 Subject: [PATCH 30/43] Fix error handling --- Objects/frameobject.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ea6735c130176a..b60d7e7fa70bff 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -456,7 +456,10 @@ framelocalsproxy_contains(PyObject *self, PyObject *key) } else { PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; if (extra != NULL) { - if (PyDict_Contains(extra, key)) { + int result = PyDict_Contains(extra, key); + if (result < 0) { + return NULL; + } else if (result > 0) { Py_RETURN_TRUE; } } From 06277f9faba85c1fcf3e36ab7b99739ca94b9177 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 28 Apr 2024 15:24:27 -0700 Subject: [PATCH 31/43] Make key index work better --- Lib/test/test_frame.py | 11 ++++++ Objects/frameobject.c | 90 +++++++++++++++++++++++++++--------------- 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 0206a47b8f04b5..6d9954bbd2dd50 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -345,6 +345,17 @@ def test_as_number(self): with self.assertRaises(TypeError): _ = d | [3] + def test_write_with_hidden(self): + def f(): + f_locals = [sys._getframe().f_locals for b in [0]][0] + f_locals['b'] = 2 + f_locals['c'] = 3 + self.assertEqual(b, 2) + self.assertEqual(c, 3) + b = 0 + c = 0 + f() + def test_repr(self): x = 1 # Introduce a reference cycle diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b60d7e7fa70bff..53c72b1da86818 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -19,29 +19,6 @@ // Utilities -static int -framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key) -{ - PyCodeObject *co = PyFrame_GetCode(frame); - // We do 2 loops here because it's highly possible the key is interned - // and we can do a pointer comparison. - for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - if (name == key) { - return i; - } - } - - for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - if (_PyUnicode_EQ(name, key)) { - return i; - } - } - - return -1; -} - // Returns borrowed reference or NULL static PyObject * framelocalsproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) @@ -74,6 +51,57 @@ framelocalsproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) return value; } +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) +{ + /* + * Returns the fast locals index of the key + * - if read == true, returns the index if the value is not NULL + * - if read == false, returns the index if the value is not hidden + */ + PyCodeObject *co = PyFrame_GetCode(frame); + int found_key = false; + + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + found_key = true; + if (read) { + if (framelocalsproxy_getval(frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + + if (!found_key) { + // This is unlikely, but we need to make sure. This means the key + // is not interned. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (_PyUnicode_EQ(name, key)) { + if (read) { + if (framelocalsproxy_getval(frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + } + + return -1; +} + static int framelocalsproxy_merge(PyObject* self, PyObject* other) { @@ -360,12 +388,11 @@ framelocalsproxy_getitem(PyObject *self, PyObject *key) PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject* co = PyFrame_GetCode(frame); - int i = framelocalsproxy_getkeyindex(frame, key); + int i = framelocalsproxy_getkeyindex(frame, key, true); if (i >= 0) { PyObject *value = framelocalsproxy_getval(frame, co, i); - if (value != NULL) { - return Py_NewRef(value); - } + assert(value != NULL); + return Py_NewRef(value); } // Okay not in the fast locals, try extra locals @@ -395,7 +422,7 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - int i = framelocalsproxy_getkeyindex(frame, key); + int i = framelocalsproxy_getkeyindex(frame, key, false); if (i >= 0) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -446,13 +473,12 @@ static PyObject* framelocalsproxy_contains(PyObject *self, PyObject *key) { PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; - int i = framelocalsproxy_getkeyindex(frame, key); + int i = framelocalsproxy_getkeyindex(frame, key, true); if (i >= 0) { PyObject *value = framelocalsproxy_getval(frame, PyFrame_GetCode(frame), i); - if (value != NULL) { - Py_RETURN_TRUE; - } + assert(value != NULL); + Py_RETURN_TRUE; } else { PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; if (extra != NULL) { From e1c3f56faf58f728618e1b07e47da80c3429f0a8 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 29 Apr 2024 21:36:35 -0700 Subject: [PATCH 32/43] Add comments for GetHiddenLocals --- Objects/frameobject.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 53c72b1da86818..8a96d54d24c19d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1868,6 +1868,10 @@ _PyFrame_GetLocals(_PyInterpreterFrame *frame) PyObject* _PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) { + /* + * This function returns all the hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ PyObject* hidden = PyDict_New(); PyFrameObject* f = _PyFrame_GetFrameObject(frame); From b672d84943a839edd094e6e80713e13bf0997a94 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 29 Apr 2024 22:01:46 -0700 Subject: [PATCH 33/43] Add global test --- Lib/test/test_frame.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 6d9954bbd2dd50..16fc4e410d3bfe 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -414,6 +414,7 @@ def test_basic(self): PyEval_GetFrameGlobals.restype = ctypes.py_object frame_globals = PyEval_GetFrameGlobals() self.assertTrue(type(frame_globals), dict) + self.assertIs(frame_globals, globals()) PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins PyEval_GetFrameBuiltins.restype = ctypes.py_object From 3e32572107133df2ef1382977524903c1a0d48a1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 1 May 2024 17:42:36 -0700 Subject: [PATCH 34/43] Remove unsupported methods --- Lib/test/test_frame.py | 6 +++--- Objects/frameobject.c | 31 ------------------------------- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 16fc4e410d3bfe..3d0bcfa96043fe 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -368,10 +368,10 @@ def test_delete(self): with self.assertRaises(TypeError): del d['x'] - with self.assertRaises(TypeError): + with self.assertRaises(AttributeError): d.clear() - with self.assertRaises(TypeError): + with self.assertRaises(AttributeError): d.pop('x') @support.cpython_only @@ -382,7 +382,7 @@ def test_sizeof(self): def test_unsupport(self): x = 1 d = sys._getframe().f_locals - with self.assertRaises(TypeError): + with self.assertRaises(AttributeError): d.copy() with self.assertRaises(TypeError): diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 8a96d54d24c19d..e659748427eab2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -585,27 +585,6 @@ framelocalsproxy_reversed(PyObject *self, PyObject *__unused) return result; } -static PyObject* -framelocalsproxy_copy_unsupported(PyObject *self, PyObject *__unused) -{ - PyErr_SetString(PyExc_TypeError, "FrameLocalsProxy is uncopyable"); - return NULL; -} - -static PyObject* -framelocalsproxy_clear_unsupported(PyObject *self, PyObject *__unused) -{ - PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); - return NULL; -} - -static PyObject* -framelocalsproxy_pop_unsupported(PyObject *self, PyObject *const *args, Py_ssize_t nargs) -{ - PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); - return NULL; -} - static PyNumberMethods framelocalsproxy_as_number = { .nb_or = framelocalsproxy_or, .nb_inplace_or = framelocalsproxy_inplace_or, @@ -624,14 +603,6 @@ static PyMethodDef framelocalsproxy_methods[] = { NULL}, {"__reversed__", framelocalsproxy_reversed, METH_NOARGS, NULL}, - {"__copy__", framelocalsproxy_copy_unsupported, METH_NOARGS, - NULL}, - {"__deepcopy__", framelocalsproxy_copy_unsupported, METH_NOARGS, - NULL}, - {"copy", framelocalsproxy_copy_unsupported, METH_NOARGS, - NULL}, - {"clear", framelocalsproxy_clear_unsupported, METH_NOARGS, - NULL}, {"keys", framelocalsproxy_keys, METH_NOARGS, NULL}, {"values", framelocalsproxy_values, METH_NOARGS, @@ -640,8 +611,6 @@ static PyMethodDef framelocalsproxy_methods[] = { NULL}, {"update", framelocalsproxy_update, METH_O, NULL}, - {"pop", _PyCFunction_CAST(framelocalsproxy_pop_unsupported), METH_FASTCALL, - NULL}, {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, NULL}, {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, From 8dc46642fa98feb1ff0b14c3da392b52e90ec971 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 1 May 2024 18:08:24 -0700 Subject: [PATCH 35/43] Support non-string keys --- Lib/test/test_frame.py | 14 +++--- Objects/frameobject.c | 104 ++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 3d0bcfa96043fe..93d0ea839d16eb 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -331,6 +331,9 @@ def test_as_dict(self): self.assertEqual(d.setdefault('new', 5), 5) self.assertEqual(d['new'], 5) + with self.assertRaises(KeyError): + d['non_exist'] + def test_as_number(self): x = 1 y = 2 @@ -345,6 +348,11 @@ def test_as_number(self): with self.assertRaises(TypeError): _ = d | [3] + def test_non_string_key(self): + d = sys._getframe().f_locals + d[1] = 2 + self.assertEqual(d[1], 2) + def test_write_with_hidden(self): def f(): f_locals = [sys._getframe().f_locals for b in [0]][0] @@ -391,12 +399,6 @@ def test_unsupport(self): with self.assertRaises(TypeError): copy.deepcopy(d) - with self.assertRaises(TypeError): - d.get(1) - - with self.assertRaises(TypeError): - d.setdefault(1, 'x') - class TestFrameCApi(unittest.TestCase): def test_basic(self): diff --git a/Objects/frameobject.c b/Objects/frameobject.c index e659748427eab2..5483a303df3d9b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -59,6 +59,9 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) * - if read == true, returns the index if the value is not NULL * - if read == false, returns the index if the value is not hidden */ + + assert(PyUnicode_CheckExact(key)); + PyCodeObject *co = PyFrame_GetCode(frame); int found_key = false; @@ -388,11 +391,13 @@ framelocalsproxy_getitem(PyObject *self, PyObject *key) PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject* co = PyFrame_GetCode(frame); - int i = framelocalsproxy_getkeyindex(frame, key, true); - if (i >= 0) { - PyObject *value = framelocalsproxy_getval(frame, co, i); - assert(value != NULL); - return Py_NewRef(value); + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } } // Okay not in the fast locals, try extra locals @@ -422,34 +427,35 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - int i = framelocalsproxy_getkeyindex(frame, key, false); - - if (i >= 0) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i >= 0) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } else if (kind & CO_FAST_CELL && oldvalue != NULL) { - if (PyCell_Check(oldvalue)) { + PyObject *oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue != NULL && PyCell_Check(oldvalue)); cell = oldvalue; + } else if (kind & CO_FAST_CELL && oldvalue != NULL) { + if (PyCell_Check(oldvalue)) { + cell = oldvalue; + } } - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); + if (cell != NULL) { + oldvalue = PyCell_GET(cell); + if (value != oldvalue) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue); + } + } else if (value != oldvalue) { + Py_XSETREF(fast[i], Py_NewRef(value)); } - } else if (value != oldvalue) { - Py_XSETREF(fast[i], Py_NewRef(value)); + Py_XDECREF(value); + return 0; } - Py_XDECREF(value); - return 0; } // Okay not in the fast locals, try extra locals @@ -473,21 +479,21 @@ static PyObject* framelocalsproxy_contains(PyObject *self, PyObject *key) { PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; - int i = framelocalsproxy_getkeyindex(frame, key, true); - - if (i >= 0) { - PyObject *value = framelocalsproxy_getval(frame, PyFrame_GetCode(frame), i); - assert(value != NULL); - Py_RETURN_TRUE; - } else { - PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; - if (extra != NULL) { - int result = PyDict_Contains(extra, key); - if (result < 0) { - return NULL; - } else if (result > 0) { - Py_RETURN_TRUE; - } + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + Py_RETURN_TRUE; + } + } + + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + int result = PyDict_Contains(extra, key); + if (result < 0) { + return NULL; + } else if (result > 0) { + Py_RETURN_TRUE; } } @@ -520,12 +526,7 @@ framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) default_value = args[1]; } - if (!PyUnicode_CheckExact(key)) { - PyErr_SetString(PyExc_TypeError, "key must be a string"); - return NULL; - } - - PyObject *result = framelocalsproxy_getitem(self, args[0]); + PyObject *result = framelocalsproxy_getitem(self, key); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) { @@ -553,11 +554,6 @@ framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t na default_value = args[1]; } - if (!PyUnicode_CheckExact(key)) { - PyErr_SetString(PyExc_TypeError, "key must be a string"); - return NULL; - } - PyObject *result = framelocalsproxy_getitem(self, key); if (result == NULL) { From 652f6416a8b603009616cc91e961da2f09ed202e Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 1 May 2024 18:14:58 -0700 Subject: [PATCH 36/43] Use static function for setitem --- Objects/frameobject.c | 187 ++++++++++++++++++++---------------------- 1 file changed, 91 insertions(+), 96 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5483a303df3d9b..b779cd4f0abf1e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ #define OFF(x) offsetof(PyFrameObject, x) -// Utilities // Returns borrowed reference or NULL static PyObject * @@ -105,6 +104,96 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) return -1; } +static PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) +{ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = PyFrame_GetCode(frame); + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +static int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + /* Merge locals into fast locals */ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject* co = PyFrame_GetCode(frame); + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return -1; + } + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i >= 0) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue != NULL && PyCell_Check(oldvalue)); + cell = oldvalue; + } else if (kind & CO_FAST_CELL && oldvalue != NULL) { + if (PyCell_Check(oldvalue)) { + cell = oldvalue; + } + } + if (cell != NULL) { + oldvalue = PyCell_GET(cell); + if (value != oldvalue) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue); + } + } else if (value != oldvalue) { + Py_XSETREF(fast[i], Py_NewRef(value)); + } + Py_XDECREF(value); + return 0; + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra == NULL) { + extra = PyDict_New(); + if (extra == NULL) { + return -1; + } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + return PyDict_SetItem(extra, key, value) < 0; +} + static int framelocalsproxy_merge(PyObject* self, PyObject* other) { @@ -134,7 +223,7 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) return -1; } - if (PyObject_SetItem(self, key, value) < 0) { + if (framelocalsproxy_setitem(self, key, value) < 0) { Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); @@ -148,8 +237,6 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) return 0; } -// Type functions - static PyObject * framelocalsproxy_keys(PyObject *self, PyObject *__unused) { @@ -301,8 +388,6 @@ framelocalsproxy_inplace_or(PyObject *self, PyObject *other) return Py_NewRef(self); } -// Methods - static PyObject* framelocalsproxy_values(PyObject *self, PyObject *__unused) { @@ -385,96 +470,6 @@ framelocalsproxy_length(PyObject *self) return size; } -static PyObject * -framelocalsproxy_getitem(PyObject *self, PyObject *key) -{ - PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject* co = PyFrame_GetCode(frame); - - if (PyUnicode_CheckExact(key)) { - int i = framelocalsproxy_getkeyindex(frame, key, true); - if (i >= 0) { - PyObject *value = framelocalsproxy_getval(frame, co, i); - assert(value != NULL); - return Py_NewRef(value); - } - } - - // Okay not in the fast locals, try extra locals - - PyObject *extra = frame->f_extra_locals; - if (extra != NULL) { - PyObject *value = PyDict_GetItem(extra, key); - if (value != NULL) { - return Py_NewRef(value); - } - } - - PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); - return NULL; -} - -static int -framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) -{ - /* Merge locals into fast locals */ - PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); - PyCodeObject* co = PyFrame_GetCode(frame); - - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); - return -1; - } - - if (PyUnicode_CheckExact(key)) { - int i = framelocalsproxy_getkeyindex(frame, key, false); - if (i >= 0) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } else if (kind & CO_FAST_CELL && oldvalue != NULL) { - if (PyCell_Check(oldvalue)) { - cell = oldvalue; - } - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } else if (value != oldvalue) { - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - return 0; - } - } - - // Okay not in the fast locals, try extra locals - - PyObject *extra = frame->f_extra_locals; - - if (extra == NULL) { - extra = PyDict_New(); - if (extra == NULL) { - return -1; - } - frame->f_extra_locals = extra; - } - - assert(PyDict_Check(extra)); - - return PyDict_SetItem(extra, key, value) < 0; -} - static PyObject* framelocalsproxy_contains(PyObject *self, PyObject *key) { From f29e6a3e6021f0123ea33c66ade1573ebffb8549 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 18:44:44 -0700 Subject: [PATCH 37/43] Fix the list comp --- Include/internal/pycore_frame.h | 3 -- Lib/test/test_listcomps.py | 5 +++ Objects/frameobject.c | 54 ++++++++++++--------------------- Python/ceval.c | 26 ++-------------- 4 files changed, 27 insertions(+), 61 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index af8a00e82a6906..a3fecb069f5dae 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -245,9 +245,6 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame); -PyObject * -_PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame); - static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) { diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 2c8eaed8d5e8e2..c09388e0437689 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -627,6 +627,11 @@ def test_frame_locals(self): import sys self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ + self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + def _recursive_replace(self, maybe_code): if not isinstance(maybe_code, types.CodeType): return maybe_code diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b779cd4f0abf1e..eb7835ba18d35a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1811,57 +1811,43 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame) -{ - PyCodeObject* co = _PyFrame_GetCode(frame); - PyFrameObject* f = _PyFrame_GetFrameObject(frame); - - if (!(co->co_flags & CO_OPTIMIZED)) { - return Py_NewRef(frame->f_locals); - } - - return _PyFrameLocalsProxy_New(f); -} - - -PyObject* -_PyFrame_GetHiddenLocals(_PyInterpreterFrame *frame) +static bool +frame_hashiddenlocals(_PyInterpreterFrame *frame) { /* * This function returns all the hidden locals introduced by PEP 709, * which are the isolated fast locals for inline comprehensions */ - PyObject* hidden = PyDict_New(); PyFrameObject* f = _PyFrame_GetFrameObject(frame); - - if (hidden == NULL) { - return NULL; - } - PyCodeObject* co = _PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - if (!(kind & CO_FAST_HIDDEN)) { - continue; + if (kind & CO_FAST_HIDDEN) { + PyObject* value = framelocalsproxy_getval(f, co, i); + + if (value != NULL) { + return true; + } } + } - PyObject* name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject* value = framelocalsproxy_getval(f, co, i); + return false; +} - if (value == NULL) { - continue; - } - if (PyDict_SetItem(hidden, name, value) < 0) { - Py_DECREF(hidden); - return NULL; - } +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame) +{ + PyCodeObject* co = _PyFrame_GetCode(frame); + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(frame)) { + return Py_NewRef(frame->f_locals); } - return hidden; + return _PyFrameLocalsProxy_New(f); } diff --git a/Python/ceval.c b/Python/ceval.c index 2c23c56771c96a..ef8b34d55a3e14 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2495,30 +2495,8 @@ _PyEval_GetFrameLocals(void) } Py_DECREF(locals); return ret; - } - - if (PyMapping_Check(locals)) { - PyObject* hidden = _PyFrame_GetHiddenLocals(current_frame); - if (hidden == NULL) { - return NULL; - } - assert(PyDict_Check(hidden)); - if (PyDict_Size(hidden) > 0) { - PyObject* ret = PyDict_New(); - if (PyDict_Update(ret, locals)) { - Py_DECREF(ret); - Py_DECREF(hidden); - return NULL; - } - if (PyDict_Update(ret, hidden)) { - Py_DECREF(ret); - Py_DECREF(hidden); - return NULL; - } - Py_DECREF(hidden); - return ret; - } - return locals; + } else if (PyDict_Check(locals)) { + return Py_XNewRef(locals); } return NULL; From e0ca4feff59f6cb1e313b22875b7bdb10fa788a5 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 19:06:26 -0700 Subject: [PATCH 38/43] Fix mapping check --- Python/ceval.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index ef8b34d55a3e14..3154d9b2b81b23 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2495,8 +2495,8 @@ _PyEval_GetFrameLocals(void) } Py_DECREF(locals); return ret; - } else if (PyDict_Check(locals)) { - return Py_XNewRef(locals); + } else if (PyMapping_Check(locals)) { + return locals; } return NULL; From f78156aaebadf57c6f640637a8c5adc076714b1a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 19:24:23 -0700 Subject: [PATCH 39/43] Fix frame_getlocals --- Objects/frameobject.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index eb7835ba18d35a..01b220b2fbc957 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -654,13 +654,7 @@ frame_getlocals(PyFrameObject *f, void *closure) } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyCodeObject* co = PyFrame_GetCode(f); - - if (!(co->co_flags & CO_OPTIMIZED)) { - return Py_NewRef(f->f_frame->f_locals); - } - - return _PyFrameLocalsProxy_New(f); + return _PyFrame_GetLocals(f->f_frame); } int @@ -1812,20 +1806,19 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, static bool -frame_hashiddenlocals(_PyInterpreterFrame *frame) +frame_hashiddenlocals(PyFrameObject *frame) { /* * This function returns all the hidden locals introduced by PEP 709, * which are the isolated fast locals for inline comprehensions */ - PyFrameObject* f = _PyFrame_GetFrameObject(frame); - PyCodeObject* co = _PyFrame_GetCode(frame); + PyCodeObject* co = PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); if (kind & CO_FAST_HIDDEN) { - PyObject* value = framelocalsproxy_getval(f, co, i); + PyObject* value = framelocalsproxy_getval(frame, co, i); if (value != NULL) { return true; @@ -1843,7 +1836,7 @@ _PyFrame_GetLocals(_PyInterpreterFrame *frame) PyCodeObject* co = _PyFrame_GetCode(frame); PyFrameObject* f = _PyFrame_GetFrameObject(frame); - if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(frame)) { + if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(f)) { return Py_NewRef(frame->f_locals); } From 4503145b5e4b02d7647417750729832ab0a81893 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 20:16:08 -0700 Subject: [PATCH 40/43] Fix test error --- Objects/frameobject.c | 62 +++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 01b220b2fbc957..75ecc6876f6fa3 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -644,17 +644,46 @@ static PyMemberDef frame_memberlist[] = { {NULL} /* Sentinel */ }; +static bool +frame_hashiddenlocals(PyFrameObject *frame) +{ + /* + * This function returns all the hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + if (kind & CO_FAST_HIDDEN) { + PyObject* value = framelocalsproxy_getval(frame, co, i); + + if (value != NULL) { + return true; + } + } + } + + return false; +} static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { + PyCodeObject *co = PyFrame_GetCode(f); + if (f == NULL) { PyErr_BadInternalCall(); return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - return _PyFrame_GetLocals(f->f_frame); + if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(f)) { + return Py_NewRef(f->f_frame->f_locals); + } + + return _PyFrameLocalsProxy_New(f); } int @@ -1805,42 +1834,13 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } -static bool -frame_hashiddenlocals(PyFrameObject *frame) -{ - /* - * This function returns all the hidden locals introduced by PEP 709, - * which are the isolated fast locals for inline comprehensions - */ - PyCodeObject* co = PyFrame_GetCode(frame); - - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - if (kind & CO_FAST_HIDDEN) { - PyObject* value = framelocalsproxy_getval(frame, co, i); - - if (value != NULL) { - return true; - } - } - } - - return false; -} - - PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame) { PyCodeObject* co = _PyFrame_GetCode(frame); PyFrameObject* f = _PyFrame_GetFrameObject(frame); - if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(f)) { - return Py_NewRef(frame->f_locals); - } - - return _PyFrameLocalsProxy_New(f); + return frame_getlocals(f, co); } From cdac22c55e57a7dbee5128efe21146f76368e8d3 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 20:40:17 -0700 Subject: [PATCH 41/43] Change the new ref for getcode --- Objects/frameobject.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 75ecc6876f6fa3..c6440d8300cd94 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -61,7 +61,7 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) assert(PyUnicode_CheckExact(key)); - PyCodeObject *co = PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); int found_key = false; // We do 2 loops here because it's highly possible the key is interned @@ -108,7 +108,7 @@ static PyObject * framelocalsproxy_getitem(PyObject *self, PyObject *key) { PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject* co = PyFrame_GetCode(frame); + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); if (PyUnicode_CheckExact(key)) { int i = framelocalsproxy_getkeyindex(frame, key, true); @@ -139,7 +139,7 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) /* Merge locals into fast locals */ PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); - PyCodeObject* co = PyFrame_GetCode(frame); + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); @@ -242,7 +242,7 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) { PyObject *names = PyList_New(0); PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject *co = PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *val = framelocalsproxy_getval(frame, co, i); @@ -393,7 +393,7 @@ framelocalsproxy_values(PyObject *self, PyObject *__unused) { PyObject *values = PyList_New(0); PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject *co = PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *value = framelocalsproxy_getval(frame, co, i); @@ -421,7 +421,7 @@ framelocalsproxy_items(PyObject *self, PyObject *__unused) { PyObject *items = PyList_New(0); PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject *co = PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); @@ -454,7 +454,7 @@ static Py_ssize_t framelocalsproxy_length(PyObject *self) { PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject *co = PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); Py_ssize_t size = 0; if (frame->f_extra_locals != NULL) { @@ -651,7 +651,7 @@ frame_hashiddenlocals(PyFrameObject *frame) * This function returns all the hidden locals introduced by PEP 709, * which are the isolated fast locals for inline comprehensions */ - PyCodeObject* co = PyFrame_GetCode(frame); + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -671,14 +671,14 @@ frame_hashiddenlocals(PyFrameObject *frame) static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { - PyCodeObject *co = PyFrame_GetCode(f); - if (f == NULL) { PyErr_BadInternalCall(); return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); + PyCodeObject *co = _PyFrame_GetCode(f->f_frame); + if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(f)) { return Py_NewRef(f->f_frame->f_locals); } @@ -1837,10 +1837,9 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame) { - PyCodeObject* co = _PyFrame_GetCode(frame); PyFrameObject* f = _PyFrame_GetFrameObject(frame); - return frame_getlocals(f, co); + return frame_getlocals(f, NULL); } From 49287ffbf1fcc96778342463d068bbf853599772 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 21:08:41 -0700 Subject: [PATCH 42/43] Avoid creating the frame object if possible --- Include/internal/pycore_frame.h | 3 ++ Objects/frameobject.c | 79 ++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a3fecb069f5dae..007c758d9c524f 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -242,6 +242,9 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); + PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c6440d8300cd94..04ad72bd847414 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -20,9 +20,9 @@ // Returns borrowed reference or NULL static PyObject * -framelocalsproxy_getval(PyFrameObject *frame, PyCodeObject *co, int i) +framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) { - PyObject **fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyObject **fast = _PyFrame_GetLocalsArray(frame); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); PyObject *value = fast[i]; @@ -71,7 +71,7 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) if (name == key) { found_key = true; if (read) { - if (framelocalsproxy_getval(frame, co, i) != NULL) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { return i; } } else { @@ -89,7 +89,7 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); if (_PyUnicode_EQ(name, key)) { if (read) { - if (framelocalsproxy_getval(frame, co, i) != NULL) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { return i; } } else { @@ -113,7 +113,7 @@ framelocalsproxy_getitem(PyObject *self, PyObject *key) if (PyUnicode_CheckExact(key)) { int i = framelocalsproxy_getkeyindex(frame, key, true); if (i >= 0) { - PyObject *value = framelocalsproxy_getval(frame, co, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); assert(value != NULL); return Py_NewRef(value); } @@ -245,7 +245,7 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *val = framelocalsproxy_getval(frame, co, i); + PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); if (val) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyList_Append(names, name); @@ -396,7 +396,7 @@ framelocalsproxy_values(PyObject *self, PyObject *__unused) PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value = framelocalsproxy_getval(frame, co, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); if (value) { PyList_Append(values, value); } @@ -425,7 +425,7 @@ framelocalsproxy_items(PyObject *self, PyObject *__unused) for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = framelocalsproxy_getval(frame, co, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); if (value) { PyObject *pair = PyTuple_Pack(2, name, value); @@ -463,7 +463,7 @@ framelocalsproxy_length(PyObject *self) } for (int i = 0; i < co->co_nlocalsplus; i++) { - if (framelocalsproxy_getval(frame, co, i) != NULL) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { size++; } } @@ -644,30 +644,6 @@ static PyMemberDef frame_memberlist[] = { {NULL} /* Sentinel */ }; -static bool -frame_hashiddenlocals(PyFrameObject *frame) -{ - /* - * This function returns all the hidden locals introduced by PEP 709, - * which are the isolated fast locals for inline comprehensions - */ - PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); - - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - if (kind & CO_FAST_HIDDEN) { - PyObject* value = framelocalsproxy_getval(frame, co, i); - - if (value != NULL) { - return true; - } - } - } - - return false; -} - static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { @@ -679,7 +655,7 @@ frame_getlocals(PyFrameObject *f, void *closure) PyCodeObject *co = _PyFrame_GetCode(f->f_frame); - if (!(co->co_flags & CO_OPTIMIZED) && !frame_hashiddenlocals(f)) { + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { return Py_NewRef(f->f_frame->f_locals); } @@ -1834,12 +1810,45 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) +{ + /* + * This function returns if there are hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = _PyFrame_GetCode(frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + if (kind & CO_FAST_HIDDEN) { + PyObject* value = framelocalsproxy_getval(frame, co, i); + + if (value != NULL) { + return true; + } + } + } + + return false; +} + + PyObject * _PyFrame_GetLocals(_PyInterpreterFrame *frame) { + // We should try to avoid creating the FrameObject if possible. + // So we check if the frame is a module or class level scope + PyCodeObject *co = _PyFrame_GetCode(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + return Py_NewRef(frame->f_locals); + } + PyFrameObject* f = _PyFrame_GetFrameObject(frame); - return frame_getlocals(f, NULL); + return _PyFrameLocalsProxy_New(f); } From 378aacf7a1cb9d4739e82ec10cc2eb52dee34a1b Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 3 May 2024 13:47:29 -0700 Subject: [PATCH 43/43] Remove a single blank line --- Objects/frameobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 04ad72bd847414..f41d94c392d9ae 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ #define OFF(x) offsetof(PyFrameObject, x) - // Returns borrowed reference or NULL static PyObject * framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i)