diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index cf0b148c2b6463a..4a7191a562cc100 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { # define _PyGC_BITS_UNREACHABLE (4) # define _PyGC_BITS_FROZEN (8) # define _PyGC_BITS_SHARED (16) +# define _PyGC_BITS_SHARED_INLINE (32) #endif /* True if the object is currently tracked by the GC. */ @@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { #ifdef Py_GIL_DISABLED -/* True if an object is shared between multiple threads and - * needs special purpose when freeing to do the possibility - * of in-flight lock-free reads occurring */ +/* True if memory the object references is shared between + * multiple threads and needs special purpose when freeing + * those references due to the possibility of in-flight + * lock-free reads occurring. The object is responsible + * for calling _PyMem_FreeDelayed on the referenced + * memory. */ static inline int _PyObject_GC_IS_SHARED(PyObject *op) { return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; } @@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { } #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) +/* True if the memory of the object is shared between multiple + * threads and needs special purpose when freeing due to + * the possibility of in-flight lock-free reads occurring. + * Objects with this bit that are GC objects will automatically + * delay-freed by PyObject_GC_Del. */ +static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { + return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0; +} +#define _PyObject_GC_IS_SHARED_INLINE(op) \ + _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op)) + +static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { + op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE; +} +#define _PyObject_GC_SET_SHARED_INLINE(op) \ + _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op)) + #endif /* Bit flags for _gc_prev */ diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1aea91abc5d69f4..dd6b0762370c926 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. extern void _PyMem_FreeDelayed(void *ptr); +// Enqueue an object to be freed possibly after some delay +extern void _PyObject_FreeDelayed(void *ptr); + // Periodically process delayed free requests. extern void _PyMem_ProcessDelayed(PyThreadState *tstate); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e7813807674abeb..4fe195b63166c17 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str) // A pointer to be freed once the QSBR read sequence reaches qsbr_goal. struct _mem_work_item { - void *ptr; + uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free uint64_t qsbr_goal; }; @@ -1084,16 +1084,27 @@ struct _mem_work_chunk { struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; }; -void -_PyMem_FreeDelayed(void *ptr) +static void +free_work_item(uintptr_t ptr) +{ + if (ptr & 0x01) { + PyObject_Free((char *)(ptr - 1)); + } + else { + PyMem_Free((void *)ptr); + } +} + +static void +free_delayed(uintptr_t ptr) { #ifndef Py_GIL_DISABLED - PyMem_Free(ptr); + free_work_item(ptr); #else if (_PyRuntime.stoptheworld.world_stopped) { // Free immediately if the world is stopped, including during // interpreter shutdown. - PyMem_Free(ptr); + free_work_item(ptr); return; } @@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr) if (buf == NULL) { // failed to allocate a buffer, free immediately _PyEval_StopTheWorld(tstate->base.interp); - PyMem_Free(ptr); + free_work_item(ptr); _PyEval_StartTheWorld(tstate->base.interp); return; } @@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr) #endif } +void +_PyMem_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed((uintptr_t)ptr); +} + +void +_PyObject_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed(((uintptr_t)ptr)|0x01); +} + static struct _mem_work_chunk * work_queue_first(struct llist_node *head) { @@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, return; } - PyMem_Free(item->ptr); + free_work_item(item->ptr); buf->rd_idx++; } @@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp) // Free the remaining items immediately. There should be no other // threads accessing the memory at this point during shutdown. struct _mem_work_item *item = &buf->array[buf->rd_idx]; - PyMem_Free(item->ptr); + free_work_item(item->ptr); buf->rd_idx++; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index c7883cd38644e17..59e76012f8fc500 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op) { size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op) } record_deallocation(_PyThreadState_GET()); - - PyObject_Free(((char *)op)-presize); + PyObject *self = (PyObject *)op; + if (_PyObject_GC_IS_SHARED_INLINE(self)) { + _PyObject_FreeDelayed(((char *)op)-presize); + } + else { + PyObject_Free(((char *)op)-presize); + } } int