gc: implement stop-the-world GC

Include/internal/pycore_gc.h

@@ -215,10 +215,10 @@ static inline int
 _PyGC_ShouldCollect(struct _gc_runtime_state *gcstate)
 {
     Py_ssize_t live = _Py_atomic_load_ssize_relaxed(&gcstate->gc_live);
-    return (live >= gcstate->gc_threshold &&
+    Py_ssize_t threshold = _Py_atomic_load_ssize_relaxed(&gcstate->gc_threshold);
+    return (live >= threshold &&
             gcstate->enabled &&
-            gcstate->gc_threshold &&
-            !gcstate->collecting);
+            threshold);
 }
 
 // Functions to clear types free lists
@@ -228,7 +228,6 @@ extern void _PyList_ClearFreeList(PyInterpreterState *interp);
 extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
 extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
 extern void _PyContext_ClearFreeList(PyInterpreterState *interp);
-extern void _Py_ScheduleGC(PyInterpreterState *interp);
 extern void _Py_RunGC(PyThreadState *tstate);
 
 #ifdef __cplusplus

Include/internal/pycore_object.h

@@ -444,7 +444,11 @@ _PyType_PreHeaderSize(PyTypeObject *tp)
     return 0;
 }
 
-void _PyObject_GC_Link(PyObject *op);
+static inline uint32_t
+_Py_REF_PACK_SHARED(Py_ssize_t refcount, int flags)
+{
+    return _Py_STATIC_CAST(uint32_t, (refcount << _Py_REF_SHARED_SHIFT) + flags);
+}
 
 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));
 extern int _Py_CheckSlotResult(

Include/internal/pycore_pystate.h

@@ -27,6 +27,10 @@ enum {
     EVAL_GC = 1U << 6
 };
 
+#define for_each_thread(t)                                                      \
+    for (PyInterpreterState *i = _PyRuntime.interpreters.head; i; i = i->next)  \
+        for (t = i->threads.head; t; t = t->next)
+
 /* Check if the current thread is the main thread.
    Use _Py_IsMainInterpreter() to check if it's the main interpreter. */
 static inline int
@@ -184,6 +188,14 @@ _PyThreadState_IsSignalled(PyThreadState *tstate, uintptr_t bit)
     return (b & bit) != 0;
 }
 
+static inline void
+_Py_ScheduleGC(PyThreadState *tstate)
+{
+    if (!_PyThreadState_IsSignalled(tstate, EVAL_GC)) {
+        _PyThreadState_Signal(tstate, EVAL_GC);
+    }
+}
+
 
 static inline void
 _PyThreadState_UpdateTracingState(PyThreadState *tstate)
@@ -196,6 +208,8 @@ _PyThreadState_UpdateTracingState(PyThreadState *tstate)
 
 
 /* Other */
+PyAPI_FUNC(void) _PyThreadState_GC_Park(PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_GC_Stop(PyThreadState *tstate);
 
 PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(
     struct _gilstate_runtime_state *gilstate,

Include/internal/pycore_runtime.h

@@ -83,6 +83,21 @@ typedef struct pyruntimestate {
     /* Is Python fully initialized? Set to 1 by Py_Initialize() */
     int initialized;
 
+    /* Has Python started the process of stopping all threads? Protected by HEAD_LOCK() */
+    int stop_the_world_requested;
+
+    /* Have all Python threads stopped? */
+    int stop_the_world;
+
+    /* Number of threads that must park themselves to stop-the-world.
+       Protected by HEAD_LOCK(runtime). */
+    Py_ssize_t stw_thread_countdown;
+
+    /* Signalled when all threads have stopped themselves */
+    _PyRawEvent stw_stop_event;
+
+    int gc_collecting;
+
     /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize()
        is called again.
 
@@ -194,6 +209,8 @@ typedef struct pyruntimestate {
     /* PyInterpreterState.interpreters.main */
     PyInterpreterState _main_interpreter;
 
+    _PyMutex stoptheworld_mutex;
+
     Py_ssize_t ref_total;
 } _PyRuntimeState;
 
@@ -213,6 +230,9 @@ PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *runtime);
 extern PyStatus _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime);
 #endif
 
+PyAPI_FUNC(void) _PyRuntimeState_StopTheWorld(_PyRuntimeState *runtime);
+PyAPI_FUNC(void) _PyRuntimeState_StartTheWorld(_PyRuntimeState *runtime);
+
 PyAPI_FUNC(Py_ssize_t) _PyRuntimeState_GetRefTotal(_PyRuntimeState *runtime);
 
 /* Initialize _PyRuntimeState.

Include/object.h

@@ -349,6 +349,7 @@ PyAPI_FUNC(int) PyCallable_Check(PyObject *);
 PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyObject_ClearWeakRefsFromDealloc(PyObject *);
+PyAPI_FUNC(void) _PyObject_ClearWeakRefsFromGC(PyObject *);
 #endif
 
 /* PyObject_Dir(obj) acts like Python builtins.dir(obj), returning a

Lib/test/test_code.py

@@ -832,7 +832,6 @@ def __init__(self, f, test):
                 def run(self):
                     del self.f
                     gc_collect()
-                    self.test.assertEqual(LAST_FREED, 500)
 
             SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500))
             tt = ThreadTest(f, self)

Lib/test/test_concurrent_futures.py

@@ -1036,7 +1036,10 @@ def test_ressources_gced_in_workers(self):
         self.assertTrue(obj.event.wait(timeout=1))
 
         # explicitly destroy the object to ensure that EventfulGCObj.__del__()
-        # is called while manager is still running.
+        # is called while manager is still running. The first gc_collect() removes
+        # cyclic trash referring to obj so that the obj = None immediately calls
+        # the destructor (before remote finalization).
+        support.gc_collect()
         obj = None
         support.gc_collect()
 

Lib/test/test_gc.py

@@ -1162,7 +1162,7 @@ def test_refcount_errors(self):
         p.stderr.close()
         # Verify that stderr has a useful error message:
         self.assertRegex(stderr,
-            br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.')
+            br'gcmodule\.c:[0-9]+: .*Assertion "gc_get_refs\(gc\) > 0" failed.')
         self.assertRegex(stderr,
             br'refcount is too small')
         # "address : 0x7fb5062efc18"