Skip to content

Commit

Permalink
gh-111916: Make hashlib related modules thread-safe without the GIL (#…
Browse files Browse the repository at this point in the history 
…111981)

Always use an individual lock on hash objects when in free-threaded builds.

Fixes #111916
  • Loading branch information
@tomasr8
tomasr8 authored Nov 15, 2023
1 parent 7218bac commit a646560
Show file tree
Hide file tree
Showing 10 changed files with 188 additions and 137 deletions.
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Make hashlib related modules thread-safe without the GIL
33 changes: 17 additions & 16 deletions Modules/_blake2/blake2b_impl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# define Py_BUILD_CORE_MODULE 1
#endif

#include <stdbool.h>
#include "Python.h"
#include "pycore_strhex.h" // _Py_strhex()

Expand All @@ -42,7 +43,8 @@ typedef struct {
PyObject_HEAD
blake2b_param param;
blake2b_state state;
PyThread_type_lock lock;
bool use_mutex;
PyMutex mutex;
} BLAKE2bObject;

#include "clinic/blake2b_impl.c.h"
Expand All @@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type)
{
BLAKE2bObject *self;
self = (BLAKE2bObject *)type->tp_alloc(type, 0);
if (self != NULL) {
self->lock = NULL;
if (self == NULL) {
return NULL;
}
HASHLIB_INIT_MUTEX(self);

return self;
}

Expand Down Expand Up @@ -278,18 +282,19 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data)

GET_BUFFER_VIEW_OR_ERROUT(data, &buf);

if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
self->lock = PyThread_allocate_lock();

if (self->lock != NULL) {
Py_BEGIN_ALLOW_THREADS
PyThread_acquire_lock(self->lock, 1);
blake2b_update(&self->state, buf.buf, buf.len);
PyThread_release_lock(self->lock);
Py_END_ALLOW_THREADS
if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
self->use_mutex = true;
}
if (self->use_mutex) {
Py_BEGIN_ALLOW_THREADS
PyMutex_Lock(&self->mutex);
blake2b_update(&self->state, buf.buf, buf.len);
PyMutex_Unlock(&self->mutex);
Py_END_ALLOW_THREADS
} else {
blake2b_update(&self->state, buf.buf, buf.len);
}

PyBuffer_Release(&buf);

Py_RETURN_NONE;
Expand Down Expand Up @@ -389,10 +394,6 @@ py_blake2b_dealloc(PyObject *self)
/* Try not to leave state in memory. */
secure_zero_memory(&obj->param, sizeof(obj->param));
secure_zero_memory(&obj->state, sizeof(obj->state));
if (obj->lock) {
PyThread_free_lock(obj->lock);
obj->lock = NULL;
}

PyTypeObject *type = Py_TYPE(self);
PyObject_Free(self);
Expand Down
33 changes: 17 additions & 16 deletions Modules/_blake2/blake2s_impl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# define Py_BUILD_CORE_MODULE 1
#endif

#include <stdbool.h>
#include "Python.h"
#include "pycore_strhex.h" // _Py_strhex()

Expand All @@ -42,7 +43,8 @@ typedef struct {
PyObject_HEAD
blake2s_param param;
blake2s_state state;
PyThread_type_lock lock;
bool use_mutex;
PyMutex mutex;
} BLAKE2sObject;

#include "clinic/blake2s_impl.c.h"
Expand All @@ -59,9 +61,11 @@ new_BLAKE2sObject(PyTypeObject *type)
{
BLAKE2sObject *self;
self = (BLAKE2sObject *)type->tp_alloc(type, 0);
if (self != NULL) {
self->lock = NULL;
if (self == NULL) {
return NULL;
}
HASHLIB_INIT_MUTEX(self);

return self;
}

Expand Down Expand Up @@ -278,18 +282,19 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data)

GET_BUFFER_VIEW_OR_ERROUT(data, &buf);

if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
self->lock = PyThread_allocate_lock();

if (self->lock != NULL) {
Py_BEGIN_ALLOW_THREADS
PyThread_acquire_lock(self->lock, 1);
blake2s_update(&self->state, buf.buf, buf.len);
PyThread_release_lock(self->lock);
Py_END_ALLOW_THREADS
if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
self->use_mutex = true;
}
if (self->use_mutex) {
Py_BEGIN_ALLOW_THREADS
PyMutex_Lock(&self->mutex);
blake2s_update(&self->state, buf.buf, buf.len);
PyMutex_Unlock(&self->mutex);
Py_END_ALLOW_THREADS
} else {
blake2s_update(&self->state, buf.buf, buf.len);
}

PyBuffer_Release(&buf);

Py_RETURN_NONE;
Expand Down Expand Up @@ -389,10 +394,6 @@ py_blake2s_dealloc(PyObject *self)
/* Try not to leave state in memory. */
secure_zero_memory(&obj->param, sizeof(obj->param));
secure_zero_memory(&obj->state, sizeof(obj->state));
if (obj->lock) {
PyThread_free_lock(obj->lock);
obj->lock = NULL;
}

PyTypeObject *type = Py_TYPE(self);
PyObject_Free(self);
Expand Down
45 changes: 18 additions & 27 deletions Modules/_hashopenssl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
# define Py_BUILD_CORE_MODULE 1
#endif

#include <stdbool.h>
#include "Python.h"
#include "pycore_hashtable.h"
#include "pycore_pyhash.h" // _Py_HashBytes()
Expand Down Expand Up @@ -227,16 +228,16 @@ typedef struct {
PyObject_HEAD
EVP_MD_CTX *ctx; /* OpenSSL message digest context */
// Prevents undefined behavior via multiple threads entering the C API.
// The lock will be NULL before threaded access has been enabled.
PyThread_type_lock lock; /* OpenSSL context lock */
bool use_mutex;
PyMutex mutex; /* OpenSSL context lock */
} EVPobject;

typedef struct {
PyObject_HEAD
HMAC_CTX *ctx; /* OpenSSL hmac context */
// Prevents undefined behavior via multiple threads entering the C API.
// The lock will be NULL before threaded access has been enabled.
PyThread_type_lock lock; /* HMAC context lock */
bool use_mutex;
PyMutex mutex; /* HMAC context lock */
} HMACobject;

#include "clinic/_hashopenssl.c.h"
Expand Down Expand Up @@ -414,8 +415,7 @@ newEVPobject(PyTypeObject *type)
if (retval == NULL) {
return NULL;
}

retval->lock = NULL;
HASHLIB_INIT_MUTEX(retval);

retval->ctx = EVP_MD_CTX_new();
if (retval->ctx == NULL) {
Expand Down Expand Up @@ -453,8 +453,6 @@ static void
EVP_dealloc(EVPobject *self)
{
PyTypeObject *tp = Py_TYPE(self);
if (self->lock != NULL)
PyThread_free_lock(self->lock);
EVP_MD_CTX_free(self->ctx);
PyObject_Free(self);
Py_DECREF(tp);
Expand Down Expand Up @@ -582,16 +580,14 @@ EVP_update(EVPobject *self, PyObject *obj)

GET_BUFFER_VIEW_OR_ERROUT(obj, &view);

if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
self->lock = PyThread_allocate_lock();
/* fail? lock = NULL and we fail over to non-threaded code. */
if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
self->use_mutex = true;
}

if (self->lock != NULL) {
if (self->use_mutex) {
Py_BEGIN_ALLOW_THREADS
PyThread_acquire_lock(self->lock, 1);
PyMutex_Lock(&self->mutex);
result = EVP_hash(self, view.buf, view.len);
PyThread_release_lock(self->lock);
PyMutex_Unlock(&self->mutex);
Py_END_ALLOW_THREADS
} else {
result = EVP_hash(self, view.buf, view.len);
Expand Down Expand Up @@ -1540,7 +1536,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
}

self->ctx = ctx;
self->lock = NULL;
HASHLIB_INIT_MUTEX(self);

if ((msg_obj != NULL) && (msg_obj != Py_None)) {
if (!_hmac_update(self, msg_obj))
Expand Down Expand Up @@ -1582,16 +1578,14 @@ _hmac_update(HMACobject *self, PyObject *obj)

GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0);

if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
self->lock = PyThread_allocate_lock();
/* fail? lock = NULL and we fail over to non-threaded code. */
if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
self->use_mutex = true;
}

if (self->lock != NULL) {
if (self->use_mutex) {
Py_BEGIN_ALLOW_THREADS
PyThread_acquire_lock(self->lock, 1);
PyMutex_Lock(&self->mutex);
r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
PyThread_release_lock(self->lock);
PyMutex_Unlock(&self->mutex);
Py_END_ALLOW_THREADS
} else {
r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
Expand Down Expand Up @@ -1633,7 +1627,7 @@ _hashlib_HMAC_copy_impl(HMACobject *self)
return NULL;
}
retval->ctx = ctx;
retval->lock = NULL;
HASHLIB_INIT_MUTEX(retval);

return (PyObject *)retval;
}
Expand All @@ -1642,9 +1636,6 @@ static void
_hmac_dealloc(HMACobject *self)
{
PyTypeObject *tp = Py_TYPE(self);
if (self->lock != NULL) {
PyThread_free_lock(self->lock);
}
HMAC_CTX_free(self->ctx);
PyObject_Free(self);
Py_DECREF(tp);
Expand Down
65 changes: 58 additions & 7 deletions Modules/clinic/md5module.c.h
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit a646560

Please sign in to comment.