pystate: move freelists to per-thread state

Include/cpython/pystate.h

@@ -106,10 +106,6 @@ typedef struct _stack_chunk {
     PyObject * data[1]; /* Variable sized */
 } _PyStackChunk;
 
-typedef struct _Py_dict_thread_state {
-    uint64_t dict_version;
-} _Py_dict_thread_state;
-
 struct mi_heap_s;
 typedef struct mi_heap_s mi_heap_t;
 typedef struct _PyEventRc _PyEventRc;
@@ -156,8 +152,6 @@ struct _ts {
      * or most recently, executing _PyEval_EvalFrameDefault. */
     _PyCFrame *cframe;
 
-    _Py_dict_thread_state dict_state;
-
     /* The thread will not stop for GC or other stop-the-world requests.
      * Used for *short* critical sections that to prevent deadlocks between
      * finalizers and stopped threads. */

Include/internal/pycore_context.h

@@ -13,7 +13,7 @@ extern PyTypeObject _PyContextTokenMissing_Type;
 /* runtime lifecycle */
 
 PyStatus _PyContext_Init(PyInterpreterState *);
-void _PyContext_Fini(PyInterpreterState *);
+void _PyContext_Fini(PyThreadState *);
 
 
 /* other API */

Include/internal/pycore_dict.h

@@ -16,7 +16,7 @@ extern "C" {
 
 /* runtime lifecycle */
 
-extern void _PyDict_Fini(PyInterpreterState *interp);
+extern void _PyDict_Fini(PyThreadState *tstate);
 
 
 /* other API */
@@ -166,14 +166,15 @@ static inline PyDictSharedKeysObject* DK_AS_SPLIT(PyDictKeysObject *dk) {
 static inline uint64_t
 _PyDict_NextVersion(PyThreadState *tstate)
 {
-    uint64_t version = tstate->dict_state.dict_version;
+    struct _Py_dict_thread_state *dict_state = &((PyThreadStateImpl *)tstate)->dict_state;
+    uint64_t version = dict_state->dict_version;
     if (version % DICT_GLOBAL_VERSION_INCREMENT == 0) {
         version = _Py_atomic_add_uint64(
             &_PyRuntime.dict_state.global_version,
             DICT_GLOBAL_VERSION_INCREMENT);
     }
     version += DICT_VERSION_INCREMENT;
-    tstate->dict_state.dict_version = version;
+    dict_state->dict_version = version;
     return version;
 }
 

Include/internal/pycore_dict_state.h

@@ -31,14 +31,18 @@ struct _Py_dict_runtime_state {
 
 typedef struct PyDictSharedKeysObject PyDictSharedKeysObject;
 
-struct _Py_dict_state {
+struct _Py_dict_thread_state {
+    uint64_t dict_version;
 #if PyDict_MAXFREELIST > 0
     /* Dictionary reuse scheme to save calls to malloc and free */
     PyDictObject *free_list[PyDict_MAXFREELIST];
     PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
     int numfree;
     int keys_numfree;
 #endif
+};
+
+struct _Py_dict_state {
     PyDict_WatchCallback watchers[DICT_MAX_WATCHERS];
     /* shared keys from deallocated types (i.e., potentially dead) */
     PyDictSharedKeysObject *tracked_shared_keys;

Include/internal/pycore_floatobject.h

@@ -13,7 +13,7 @@ extern "C" {
 
 extern void _PyFloat_InitState(PyInterpreterState *);
 extern PyStatus _PyFloat_InitTypes(PyInterpreterState *);
-extern void _PyFloat_Fini(PyInterpreterState *);
+extern void _PyFloat_Fini(PyThreadState *);
 extern void _PyFloat_FiniType(PyInterpreterState *);
 
 

Include/internal/pycore_gc.h

@@ -222,12 +222,12 @@ _PyGC_ShouldCollect(struct _gc_runtime_state *gcstate)
 }
 
 // Functions to clear types free lists
-extern void _PyTuple_ClearFreeList(PyInterpreterState *interp);
-extern void _PyFloat_ClearFreeList(PyInterpreterState *interp);
-extern void _PyList_ClearFreeList(PyInterpreterState *interp);
-extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
-extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
-extern void _PyContext_ClearFreeList(PyInterpreterState *interp);
+extern void _PyTuple_ClearFreeList(PyThreadState *tstate);
+extern void _PyFloat_ClearFreeList(PyThreadState *tstate);
+extern void _PyList_ClearFreeList(PyThreadState *tstate);
+extern void _PyDict_ClearFreeList(PyThreadState *tstate);
+extern void _PyAsyncGen_ClearFreeLists(PyThreadState *tstate);
+extern void _PyContext_ClearFreeList(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);
 
 #ifdef __cplusplus

Include/internal/pycore_genobject.h

@@ -14,7 +14,7 @@ extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *);
 
 /* runtime lifecycle */
 
-extern void _PyAsyncGen_Fini(PyInterpreterState *);
+extern void _PyAsyncGen_Fini(PyThreadState *);
 
 
 /* other API */

Include/internal/pycore_interp.h

@@ -68,6 +68,13 @@ typedef struct PyThreadStateImpl {
     // semi-public fields are in PyThreadState
     PyThreadState tstate;
 
+    struct _Py_float_state float_state;
+    struct _Py_tuple_state tuple;
+    struct _Py_list_state list;
+    struct _Py_dict_thread_state dict_state;
+    struct _Py_async_gen_state async_gen;
+    struct _Py_context_state context;
+
     struct brc_state brc;
 
     struct qsbr *qsbr;
@@ -194,14 +201,9 @@ struct _is {
     uint8_t active_code_watchers;
 
     struct _Py_unicode_state unicode;
-    struct _Py_float_state float_state;
     struct _Py_long_state long_state;
 
-    struct _Py_tuple_state tuple;
-    struct _Py_list_state list;
     struct _Py_dict_state dict_state;
-    struct _Py_async_gen_state async_gen;
-    struct _Py_context_state context;
     struct _Py_exc_state exc_state;
 
     struct ast_state ast;

Include/internal/pycore_list.h

@@ -14,7 +14,7 @@ extern "C" {
 
 /* runtime lifecycle */
 
-extern void _PyList_Fini(PyInterpreterState *);
+extern void _PyList_Fini(PyThreadState *);
 
 
 /* other API */

Include/internal/pycore_pystate.h

@@ -106,6 +106,11 @@ _PyThreadState_GET(void)
     return _Py_current_tstate;
 #endif
 }
+static inline PyThreadStateImpl*
+_PyThreadStateImpl_GET(void)
+{
+    return (PyThreadStateImpl *)_PyThreadState_GET();
+}
 
 static inline void
 _PyThreadState_SET(PyThreadState *tstate)

Include/internal/pycore_tuple.h

@@ -15,7 +15,7 @@ extern "C" {
 
 extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *);
 extern PyStatus _PyTuple_InitTypes(PyInterpreterState *);
-extern void _PyTuple_Fini(PyInterpreterState *);
+extern void _PyTuple_Fini(PyThreadState *);
 
 
 /* other API */

Modules/gcmodule.c

@@ -1359,20 +1359,32 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate,
     }
 }
 
+static void
+clear_freelists(PyThreadState *tstate)
+{
+    _PyTuple_ClearFreeList(tstate);
+    _PyFloat_ClearFreeList(tstate);
+    _PyList_ClearFreeList(tstate);
+    _PyDict_ClearFreeList(tstate);
+    _PyAsyncGen_ClearFreeLists(tstate);
+    _PyContext_ClearFreeList(tstate);
+}
+
 /* Clear all free lists
  * All free lists are cleared during the collection of the highest generation.
  * Allocated items in the free list may keep a pymalloc arena occupied.
  * Clearing the free lists may give back memory to the OS earlier.
  */
 static void
-clear_freelists(PyInterpreterState *interp)
+clear_all_freelists(PyInterpreterState *interp)
 {
-    _PyTuple_ClearFreeList(interp);
-    _PyFloat_ClearFreeList(interp);
-    _PyList_ClearFreeList(interp);
-    _PyDict_ClearFreeList(interp);
-    _PyAsyncGen_ClearFreeLists(interp);
-    _PyContext_ClearFreeList(interp);
+    HEAD_LOCK(&_PyRuntime);
+    PyThreadState *tstate = interp->threads.head;
+    while (tstate != NULL) {
+        clear_freelists(tstate);
+        tstate = tstate->next;
+    }
+    HEAD_UNLOCK(&_PyRuntime);
 }
 
 /* Deduce which objects among "base" are unreachable from outside the list
@@ -1644,6 +1656,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
     PyGC_Head final_unreachable;
     handle_resurrected_objects(&unreachable, &final_unreachable);
 
+    /* Clear free list only during the collection of the highest
+     * generation */
+    if (generation == NUM_GENERATIONS-1) {
+        clear_all_freelists(tstate->interp);
+    }
+
     _PyRuntimeState_StartTheWorld(&_PyRuntime);
 
     /* Call tp_clear on objects in the final_unreachable set.  This will cause
@@ -1653,6 +1671,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
     m += gc_list_size(&final_unreachable);
     delete_garbage(tstate, gcstate, &final_unreachable);
 
+    if (reason == GC_REASON_MANUAL) {
+        // Clear this thread's freelists again after deleting garbage
+        // for more precise block accounting when calling gc.collect().
+        clear_freelists(tstate);
+    }
+
     /* Collect statistics on uncollectable objects found and print
      * debugging information. */
     for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) {
@@ -1673,12 +1697,6 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
      */
     handle_legacy_finalizers(tstate, gcstate, &finalizers);
 
-    /* Clear free list only during the collection of the highest
-     * generation */
-    if (generation == NUM_GENERATIONS-1) {
-        clear_freelists(tstate->interp);
-    }
-
     if (_PyErr_Occurred(tstate)) {
         if (reason == GC_REASON_SHUTDOWN) {
             _PyErr_Clear(tstate);

Objects/dictobject.c

@@ -246,12 +246,17 @@ get_dict_state(void)
     return &interp->dict_state;
 }
 
+static struct _Py_dict_thread_state *
+get_dict_thread_state(void)
+{
+    return &_PyThreadStateImpl_GET()->dict_state;
+}
 
 void
-_PyDict_ClearFreeList(PyInterpreterState *interp)
+_PyDict_ClearFreeList(PyThreadState *tstate)
 {
 #if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = &interp->dict_state;
+    struct _Py_dict_thread_state *state = &((PyThreadStateImpl *)tstate)->dict_state;
     while (state->numfree) {
         PyDictObject *op = state->free_list[--state->numfree];
         assert(PyDict_CheckExact(op));
@@ -263,17 +268,16 @@ _PyDict_ClearFreeList(PyInterpreterState *interp)
 #endif
 }
 
-
 void
-_PyDict_Fini(PyInterpreterState *interp)
+_PyDict_Fini(PyThreadState *tstate)
 {
-    _PyDict_ClearFreeList(interp);
-    struct _Py_dict_state *state = &interp->dict_state;
+    _PyDict_ClearFreeList(tstate);
 #if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0
+    struct _Py_dict_thread_state *state = &((PyThreadStateImpl *)tstate)->dict_state;
     state->numfree = -1;
     state->keys_numfree = -1;
 #endif
-    PyDictSharedKeysObject **ptr = &state->tracked_shared_keys;
+    PyDictSharedKeysObject **ptr = &tstate->interp->dict_state.tracked_shared_keys;
     PyDictSharedKeysObject *value;
     while ((value = *ptr) != NULL) {
         *ptr = value->next;
@@ -293,7 +297,7 @@ void
 _PyDict_DebugMallocStats(FILE *out)
 {
 #if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state();
+    struct _Py_dict_thread_state *state = get_dict_thread_state();
     _PyDebugAllocatorStats(out, "free PyDictObject",
                            state->numfree, sizeof(PyDictObject));
 #endif
@@ -636,8 +640,7 @@ new_keys_object(uint8_t log2_size, bool unicode)
     }
 
 #if PyDict_MAXFREELIST > 0
-    PyThreadState *tstate = _PyThreadState_GET();
-    struct _Py_dict_state *state = &tstate->interp->dict_state;
+    struct _Py_dict_thread_state *state = get_dict_thread_state();
 #ifdef Py_DEBUG
     // new_keys_object() must not be called after _PyDict_Fini()
     assert(state->keys_numfree != -1);
@@ -700,7 +703,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
         return;
     }
 #if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state();
+    struct _Py_dict_thread_state *state = get_dict_thread_state();
 #ifdef Py_DEBUG
     // free_keys_object() must not be called after _PyDict_Fini()
     assert(state->keys_numfree != -1);
@@ -760,7 +763,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free
     PyDictObject *mp;
     assert(keys != NULL);
 #if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state();
+    struct _Py_dict_thread_state *state = get_dict_thread_state();
 #ifdef Py_DEBUG
     // new_dict() must not be called after _PyDict_Fini()
     assert(state->numfree != -1);
@@ -1809,7 +1812,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode)
     ASSERT_CONSISTENT(mp);
     if (oldkeys != Py_EMPTY_KEYS && oldkeys->dk_kind != DICT_KEYS_SPLIT) {
 #if PyDict_MAXFREELIST > 0
-        struct _Py_dict_state *state = get_dict_state();
+        struct _Py_dict_thread_state *state = get_dict_thread_state();
 #ifdef Py_DEBUG
         // dictresize() must not be called after _PyDict_Fini()
         assert(state->keys_numfree != -1);
@@ -2703,7 +2706,7 @@ dict_dealloc(PyDictObject *mp)
         free_keys_object(keys, false);
     }
 #if PyDict_MAXFREELIST > 0
-    struct _Py_dict_state *state = get_dict_state();
+    struct _Py_dict_thread_state *state = get_dict_thread_state();
 #ifdef Py_DEBUG
     // new_dict() must not be called after _PyDict_Fini()
     assert(state->numfree != -1);

Objects/floatobject.c

@@ -34,8 +34,7 @@ class float "PyObject *" "&PyFloat_Type"
 static struct _Py_float_state *
 get_float_state(void)
 {
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    return &interp->float_state;
+    return &_PyThreadStateImpl_GET()->float_state;
 }
 #endif
 
@@ -2012,10 +2011,10 @@ _PyFloat_InitTypes(PyInterpreterState *interp)
 }
 
 void
-_PyFloat_ClearFreeList(PyInterpreterState *interp)
+_PyFloat_ClearFreeList(PyThreadState *tstate)
 {
 #if PyFloat_MAXFREELIST > 0
-    struct _Py_float_state *state = &interp->float_state;
+    struct _Py_float_state *state = &((PyThreadStateImpl *)tstate)->float_state;
     PyFloatObject *f = state->free_list;
     while (f != NULL) {
         PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
@@ -2028,11 +2027,11 @@ _PyFloat_ClearFreeList(PyInterpreterState *interp)
 }
 
 void
-_PyFloat_Fini(PyInterpreterState *interp)
+_PyFloat_Fini(PyThreadState *tstate)
 {
-    _PyFloat_ClearFreeList(interp);
+    _PyFloat_ClearFreeList(tstate);
 #if defined(Py_DEBUG) && PyFloat_MAXFREELIST > 0
-    struct _Py_float_state *state = &interp->float_state;
+    struct _Py_float_state *state = &((PyThreadStateImpl *)tstate)->float_state;
     state->numfree = -1;
 #endif
 }