parking_lot: add mutexes and one-time notifications

This adds a recursive lock that will be used for locking I/O streams.
The lock performs a direct handoff if the waiting thread has been paused
for at least 1 ms. Otherwise, the lock supports barging.

The design is inspired by WTF locks (WebKit) and locking in Go.
See https://webkit.org/blog/6161/locking-in-webkit/

The "parking lot" implementation will likely need to be improved before
release. Currently, it uses a fixed-size hashtable (251 taken from Go)
and the same linked-list for waiters and collisions. Note that Go uses
a treap (random binary serach tree) for collisions, while WebKit resizes
the hashtable to ensure 3x as many buckets as threads.

Include/Python.h

@@ -46,6 +46,7 @@
 #include "typeslots.h"
 #include "pyhash.h"
 #include "cpython/pydebug.h"
+#include "cpython/lock.h"
 #include "bytearrayobject.h"
 #include "bytesobject.h"
 #include "unicodeobject.h"

Include/cpython/lock.h

@@ -0,0 +1,137 @@
+#ifndef Py_LIMITED_API
+#ifndef Py_LOCK_H
+#define Py_LOCK_H
+
+#include "pyatomic.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    uintptr_t v;
+} _PyOnceFlag;
+
+typedef struct {
+    uint8_t v;
+} _PyMutex;
+
+typedef _PyMutex PyMutex;
+
+typedef struct {
+    uintptr_t v;
+    size_t recursions;
+} _PyRecursiveMutex;
+
+typedef enum {
+    UNLOCKED = 0,
+    LOCKED = 1,
+    HAS_PARKED = 2,
+    ONCE_INITIALIZED = 4,
+    THREAD_ID_MASK = ~(LOCKED | HAS_PARKED)
+} _PyMutex_State;
+
+PyAPI_FUNC(void) _PyMutex_lock_slow(_PyMutex *m);
+PyAPI_FUNC(void) _PyMutex_unlock_slow(_PyMutex *m);
+PyAPI_FUNC(int) _PyMutex_TryLockSlow(_PyMutex *m);
+
+PyAPI_FUNC(void) _PyRecursiveMutex_lock_slow(_PyRecursiveMutex *m);
+PyAPI_FUNC(void) _PyRecursiveMutex_unlock_slow(_PyRecursiveMutex *m);
+
+PyAPI_FUNC(int) _PyBeginOnce_slow(_PyOnceFlag *o);
+PyAPI_FUNC(void) _PyEndOnce(_PyOnceFlag *o);
+PyAPI_FUNC(void) _PyEndOnceFailed(_PyOnceFlag *o);
+
+static inline int
+_PyMutex_is_locked(_PyMutex *m)
+{
+    return _Py_atomic_load_uint8(&m->v) & LOCKED;
+}
+
+static inline int
+_PyMutex_lock_fast(_PyMutex *m)
+{
+    return _Py_atomic_compare_exchange_uint8(&m->v, UNLOCKED, LOCKED);
+}
+
+static inline void
+_PyMutex_lock(_PyMutex *m)
+{
+    if (_PyMutex_lock_fast(m)) {
+        return;
+    }
+    _PyMutex_lock_slow(m);
+}
+
+static inline int
+_PyMutex_TryLock(_PyMutex *m)
+{
+    if (_PyMutex_lock_fast(m)) {
+        return 1;
+    }
+    return _PyMutex_TryLockSlow(m);
+}
+
+static inline int
+_PyMutex_unlock_fast(_PyMutex *m)
+{
+    return _Py_atomic_compare_exchange_uint8(&m->v, LOCKED, UNLOCKED);
+}
+
+static inline void
+_PyMutex_unlock(_PyMutex *m)
+{
+    if (_PyMutex_unlock_fast(m)) {
+        return;
+    }
+    _PyMutex_unlock_slow(m);
+}
+
+static inline void
+_PyRecursiveMutex_lock(_PyRecursiveMutex *m)
+{
+    if (_Py_atomic_compare_exchange_uintptr(&m->v, UNLOCKED, _Py_ThreadId() | LOCKED)) {
+        return;
+    }
+    _PyRecursiveMutex_lock_slow(m);
+}
+
+static inline int
+_PyRecursiveMutex_owns_lock(_PyRecursiveMutex *m)
+{
+    uintptr_t v = _Py_atomic_load_uintptr(&m->v);
+    return (v & THREAD_ID_MASK) == _Py_ThreadId();
+}
+
+static inline void
+_PyRecursiveMutex_unlock(_PyRecursiveMutex *m)
+{
+    uintptr_t v = _Py_atomic_load_uintptr_relaxed(&m->v);
+    if (m->recursions == 0 && (v & 3) == LOCKED) {
+        if (_Py_atomic_compare_exchange_uintptr(&m->v, v, UNLOCKED)) {
+            return;
+        }
+    }
+    _PyRecursiveMutex_unlock_slow(m);
+}
+
+static inline int
+_PyOnce_Initialized(_PyOnceFlag *o)
+{
+    return (_Py_atomic_load_uintptr(&o->v) & ONCE_INITIALIZED) != 0;
+}
+
+static inline int
+_PyBeginOnce(_PyOnceFlag *o)
+{
+    if ((_Py_atomic_load_uintptr(&o->v) & ONCE_INITIALIZED) != 0) {
+        return 0;
+    }
+    return _PyBeginOnce_slow(o);
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_LOCK_H */
+#endif /* Py_LIMITED_API */

Include/internal/pycore_critical_section.h

@@ -0,0 +1,194 @@
+#ifndef Py_LIMITED_API
+#ifndef Py_INTERNAL_CRITICAL_SECTION_H
+#define Py_INTERNAL_CRITICAL_SECTION_H
+
+#include "pycore_pystate.h"
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+/*
+ * Implementation of Python critical sections.
+ *
+ * Python critical sections are helpers to replace the global interpreter lock
+ * with finer grained locking. A Python critical section is a region of code
+ * that can only be executed by a single thread at at time. The regions begin
+ * with a call to _Py_critical_section_begin and end with either an explicit
+ * call to _Py_critical_section_end or *implicitly* at any point that might
+ * have  released the global interpreter lock. This is a substantial difference
+ * from the traditonal notion of a "critical section", where the end of the
+ * section is typically explicitly marked.
+ *
+ * The critical section can be resumed after a potential implicit ending by
+ * the _Py_critical_section_resume function.
+ *
+ * The purposed of the implicitly ending critical sections is to avoid
+ * potential deadlock due to holding locks on multiple objects. Any time a
+ * thread would have released the GIL, it releases all locks from critical
+ * sections. This includes block on a lock acquisition.
+ *
+ * The following are examples of calls that may implicitly end a critical
+ * section:
+ *
+ *  Py_DECREF, PyObject_GC_New, PyObject_Call, PyObject_RichCompareBool,
+ *  Py_BuildValue, _Py_critical_section_begin
+ *
+ * The following are examples of calls that do NOT implicitly end a critical
+ * section:
+ *
+ *  Py_INCREF, PyMem_RawMalloc, PyMem_RawFree, memset and other C functions
+ *  that do not call into the Python API.
+ */
+
+enum {
+    // Tag to used with `prev` pointer to differentiate _Py_critical_section
+    // vs. _Py_critical_section2.
+    _Py_CRITICAL_SECTION_INACTIVE = 1,
+
+    _Py_CRITICAL_SECTION_TWO_MUTEXES = 2,
+
+    _Py_CRITICAL_SECTION_MASK = 3
+};
+
+#define Py_BEGIN_CRITICAL_SECTION(m) {          \
+    struct _Py_critical_section _cs;            \
+    _Py_critical_section_begin(&_cs, m)
+
+#define Py_END_CRITICAL_SECTION                 \
+    _Py_critical_section_end(&_cs);             \
+}
+
+#define Py_BEGIN_CRITICAL_SECTION2(m1, m2) {    \
+    struct _Py_critical_section2 _cs2;          \
+    _Py_critical_section2_begin(&_cs2, m1, m2)
+
+#define Py_END_CRITICAL_SECTION2                \
+    _Py_critical_section2_end(&_cs2);           \
+}
+
+struct _Py_critical_section {
+    // Pointer to the an outer active critical section (or
+    // _Py_critical_section_sentinel). The two least-significant-bits indicate
+    // whether the pointed-to critical section is inactive and whether it is
+    // a _Py_critical_section2 object.
+    uintptr_t prev;
+
+    // Mutex used to protect critical section
+    _PyMutex *mutex;
+};
+
+// A critical section protected by two mutexes. Use
+// _Py_critical_section2_begin and _Py_critical_section2_end.
+struct _Py_critical_section2 {
+    struct _Py_critical_section base;
+
+    _PyMutex *mutex2;
+};
+
+static inline int
+_Py_critical_section_is_active(uintptr_t tag)
+{
+    return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0;
+}
+
+PyAPI_FUNC(void)
+_Py_critical_section_resume(PyThreadState *tstate);
+
+PyAPI_FUNC(void)
+_Py_critical_section_begin_slow(struct _Py_critical_section *c, _PyMutex *m);
+
+PyAPI_FUNC(void)
+_Py_critical_section2_begin_slow(struct _Py_critical_section2 *c,
+                                 _PyMutex *m1, _PyMutex *m2, int flag);
+
+static inline void
+_Py_critical_section_begin(struct _Py_critical_section *c, _PyMutex *m)
+{
+    if (_PyMutex_lock_fast(m)) {
+        PyThreadState *tstate = PyThreadState_GET();
+        c->mutex = m;
+        c->prev = tstate->critical_section;
+        tstate->critical_section = (uintptr_t)c;
+    }
+    else {
+        _Py_critical_section_begin_slow(c, m);
+    }
+}
+
+static inline void
+_Py_critical_section_pop(struct _Py_critical_section *c)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    uintptr_t prev = c->prev;
+    tstate->critical_section = prev;
+
+    if (_PY_UNLIKELY((prev & _Py_CRITICAL_SECTION_INACTIVE))) {
+        _Py_critical_section_resume(tstate);
+    }
+}
+
+static inline void
+_Py_critical_section_end(struct _Py_critical_section *c)
+{
+    _PyMutex_unlock(c->mutex);
+    _Py_critical_section_pop(c);
+}
+
+static inline void
+_Py_critical_section2_begin(struct _Py_critical_section2 *c,
+                            _PyMutex *m1, _PyMutex *m2)
+{
+    if ((uintptr_t)m2 < (uintptr_t)m1) {
+        _PyMutex *m1_ = m1;
+        m1 = m2;
+        m2 = m1_;
+    }
+    else if (m1 == m2) {
+        c->mutex2 = NULL;
+        _Py_critical_section_begin(&c->base, m1);
+        return;
+    }
+    if (_PyMutex_lock_fast(m1)) {
+        if (_PyMutex_lock_fast(m2)) {
+            PyThreadState *tstate = PyThreadState_GET();
+            c->base.mutex = m1;
+            c->mutex2 = m2;
+            c->base.prev = tstate->critical_section;
+
+            uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES;
+            tstate->critical_section = p;
+        }
+        else {
+            _Py_critical_section2_begin_slow(c, m1, m2, 1);
+        }
+    }
+    else {
+        _Py_critical_section2_begin_slow(c, m1, m2, 0);
+    }
+}
+
+static inline void
+_Py_critical_section2_end(struct _Py_critical_section2 *c)
+{
+    if (c->mutex2) {
+        _PyMutex_unlock(c->mutex2);
+    }
+    _PyMutex_unlock(c->base.mutex);
+    _Py_critical_section_pop(&c->base);
+}
+
+PyAPI_FUNC(void)
+_Py_critical_section_end_all(PyThreadState *tstate);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_CRITICAL_SECTION_H */
+#endif /* !Py_LIMITED_API */

Include/internal/pycore_gc.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_lock.h"
+
 /* GC information is stored BEFORE the object structure. */
 typedef struct {
     // Pointer to next object in the list.