Skip to content

Commit

Permalink
pythongh-95174: Add pthread stubs for WASI
Browse files Browse the repository at this point in the history
Neither WASI nor default builds with Emscripten have support for
threading. While Emscripten comes with pthread stubs, WASI-SDK 16 and
earlier are lacking stubs.

Python initially used WASIX stubs to provide pthread stubs. This
changeset introduces stubs modelled after Emscripten's stubs.

Emscripten without pthread emulation now has ``sys.thread_info.name``
``pthread-stubs``, too.
  • Loading branch information
tiran committed Jul 27, 2022
1 parent 2833f37 commit d707f52
Show file tree
Hide file tree
Showing 14 changed files with 327 additions and 30 deletions.
2 changes: 2 additions & 0 deletions Doc/library/sys.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1658,6 +1658,8 @@ always available.
| | |
| | * ``'nt'``: Windows threads |
| | * ``'pthread'``: POSIX threads |
| | * ``'pthread-stubs'``: stub POSIX threads |
| | (on WebAssembly platforms without threading support) |
| | * ``'solaris'``: Solaris threads |
+------------------+---------------------------------------------------------+
| :const:`lock` | Name of the lock implementation: |
Expand Down
88 changes: 88 additions & 0 deletions Include/cpython/pthread_stubs.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#ifndef Py_CPYTHON_PTRHEAD_STUBS_H
#define Py_CPYTHON_PTRHEAD_STUBS_H

#if !defined(HAVE_PTHREAD_STUBS)
# error "this header file requires stubbed pthreads."
#endif

#ifndef _POSIX_THREADS
# define _POSIX_THREADS 1
#endif

/* Minimal pthread stubs for CPython
*
* The stubs implement a bare minimum pthread API for CPython.
* - pthread_create() fails.
* - pthread_exit() calls exit(0).
* - pthread_key_*() functions implement minimal TSS without destructor.
* - all other functions do nothing and return 0.
*/

#ifdef __wasi__
// WASI's bits/alltypes.h provides type definitions when __NEED_ is set.
// The header file can be included multiple times.
# define __NEED_pthread_cond_t 1
# define __NEED_pthread_condattr_t 1
# define __NEED_pthread_mutex_t 1
# define __NEED_pthread_mutexattr_t 1
# define __NEED_pthread_key_t 1
# define __NEED_pthread_t 1
# define __NEED_pthread_attr_t 1
# include <bits/alltypes.h>
#else
typedef struct { void *__x; } pthread_cond_t;
typedef struct { unsigned __attr; } pthread_condattr_t;
typedef struct { void *__x; } pthread_mutex_t;
typedef struct { unsigned __attr; } pthread_mutexattr_t;
typedef unsigned pthread_key_t;
typedef unsigned pthread_t;
typedef struct { unsigned __attr; } pthread_attr_t;
#endif

// mutex
PyAPI_FUNC(int) pthread_mutex_init(pthread_mutex_t *restrict mutex,
const pthread_mutexattr_t *restrict attr);
PyAPI_FUNC(int) pthread_mutex_destroy(pthread_mutex_t *mutex);
PyAPI_FUNC(int) pthread_mutex_trylock(pthread_mutex_t *mutex);
PyAPI_FUNC(int) pthread_mutex_lock(pthread_mutex_t *mutex);
PyAPI_FUNC(int) pthread_mutex_unlock(pthread_mutex_t *mutex);

// condition
PyAPI_FUNC(int) pthread_cond_init(pthread_cond_t *restrict cond,
const pthread_condattr_t *restrict attr);
PyAPI_FUNC(int) pthread_cond_destroy(pthread_cond_t *cond);
PyAPI_FUNC(int) pthread_cond_wait(pthread_cond_t *restrict cond,
pthread_mutex_t *restrict mutex);
PyAPI_FUNC(int) pthread_cond_timedwait(pthread_cond_t *restrict cond,
pthread_mutex_t *restrict mutex,
const struct timespec *restrict abstime);
PyAPI_FUNC(int) pthread_cond_signal(pthread_cond_t *cond);
PyAPI_FUNC(int) pthread_condattr_init(pthread_condattr_t *attr);
PyAPI_FUNC(int) pthread_condattr_setclock(
pthread_condattr_t *attr, clockid_t clock_id);

// pthread
PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread,
const pthread_attr_t *restrict attr,
void *(*start_routine)(void *),
void *restrict arg);
PyAPI_FUNC(int) pthread_detach(pthread_t thread);
PyAPI_FUNC(pthread_t) pthread_self(void);
PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__));
PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr);
PyAPI_FUNC(int) pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize);
PyAPI_FUNC(int) pthread_attr_destroy(pthread_attr_t *attr);


// pthread_key
#ifndef PTHREAD_KEYS_MAX
# define PTHREAD_KEYS_MAX 128
#endif

PyAPI_FUNC(int) pthread_key_create(pthread_key_t *key,
void (*destr_function)(void *));
PyAPI_FUNC(int) pthread_key_delete(pthread_key_t key);
PyAPI_FUNC(void *) pthread_getspecific(pthread_key_t key);
PyAPI_FUNC(int) pthread_setspecific(pthread_key_t key, const void *value);

#endif // Py_CPYTHON_PTRHEAD_STUBS_H
3 changes: 3 additions & 0 deletions Include/cpython/pythread.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock);
but hardcode the unsigned long to avoid errors for include directive.
*/
# define NATIVE_TSS_KEY_T unsigned long
#elif defined(HAVE_PTHREAD_STUBS)
# include "cpython/pthread_stubs.h"
# define NATIVE_TSS_KEY_T pthread_key_t
#else
# error "Require native threads. See https://bugs.python.org/issue31370"
#endif
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_sys.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ def test_attributes(self):
def test_thread_info(self):
info = sys.thread_info
self.assertEqual(len(info), 3)
self.assertIn(info.name, ('nt', 'pthread', 'solaris', None))
self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None))
self.assertIn(info.lock, ('semaphore', 'mutex+cond', None))

@unittest.skipUnless(support.is_emscripten, "only available on Emscripten")
Expand Down
1 change: 1 addition & 0 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1544,6 +1544,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/cpython/objimpl.h \
$(srcdir)/Include/cpython/odictobject.h \
$(srcdir)/Include/cpython/picklebufobject.h \
$(srcdir)/Include/cpython/pthread_stubs.h \
$(srcdir)/Include/cpython/pyctype.h \
$(srcdir)/Include/cpython/pydebug.h \
$(srcdir)/Include/cpython/pyerrors.h \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
wasm32-wasi builds no longer depend on WASIX's pthread stubs. Python now has
its own stubbed pthread API.
18 changes: 13 additions & 5 deletions Python/thread.c
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,15 @@ PyThread_init_thread(void)
PyThread__init_thread();
}

#if defined(_POSIX_THREADS)
# define PYTHREAD_NAME "pthread"
#if defined(HAVE_PTHREAD_STUBS)
# define PYTHREAD_NAME "pthread-stubs"
# include "thread_pthread_stubs.h"
#elif defined(_POSIX_THREADS)
# if defined(__EMSCRIPTEN__) || !defined(__EMSCRIPTEN_PTHREADS__)
# define PYTHREAD_NAME "pthread-stubs"
# else
# define PYTHREAD_NAME "pthread"
# endif
# include "thread_pthread.h"
#elif defined(NT_THREADS)
# define PYTHREAD_NAME "nt"
Expand Down Expand Up @@ -171,7 +178,9 @@ PyThread_GetInfo(void)
}
PyStructSequence_SET_ITEM(threadinfo, pos++, value);

#ifdef _POSIX_THREADS
#ifdef HAVE_PTHREAD_STUBS
value = Py_NewRef(Py_None);
#elif defined(_POSIX_THREADS)
#ifdef USE_SEMAPHORES
value = PyUnicode_FromString("semaphore");
#else
Expand All @@ -182,8 +191,7 @@ PyThread_GetInfo(void)
return NULL;
}
#else
Py_INCREF(Py_None);
value = Py_None;
value = Py_NewRef(Py_None);
#endif
PyStructSequence_SET_ITEM(threadinfo, pos++, value);

Expand Down
4 changes: 3 additions & 1 deletion Python/thread_pthread.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
#if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
#define destructor xxdestructor
#endif
#include <pthread.h>
#ifndef HAVE_PTHREAD_STUBS
# include <pthread.h>
#endif
#if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
#undef destructor
#endif
Expand Down
185 changes: 185 additions & 0 deletions Python/thread_pthread_stubs.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
#include "cpython/pthread_stubs.h"

// mutex
int
pthread_mutex_init(pthread_mutex_t *restrict mutex,
const pthread_mutexattr_t *restrict attr)
{
return 0;
}

int
pthread_mutex_destroy(pthread_mutex_t *mutex)
{
return 0;
}

int
pthread_mutex_trylock(pthread_mutex_t *mutex)
{
return 0;
}

int
pthread_mutex_lock(pthread_mutex_t *mutex)
{
return 0;
}

int
pthread_mutex_unlock(pthread_mutex_t *mutex)
{
return 0;
}

// condition
int
pthread_cond_init(pthread_cond_t *restrict cond,
const pthread_condattr_t *restrict attr)
{
return 0;
}

PyAPI_FUNC(int)pthread_cond_destroy(pthread_cond_t *cond)
{
return 0;
}

int
pthread_cond_wait(pthread_cond_t *restrict cond,
pthread_mutex_t *restrict mutex)
{
return 0;
}

int
pthread_cond_timedwait(pthread_cond_t *restrict cond,
pthread_mutex_t *restrict mutex,
const struct timespec *restrict abstime)
{
return 0;
}

int
pthread_cond_signal(pthread_cond_t *cond)
{
return 0;
}

int
pthread_condattr_init(pthread_condattr_t *attr)
{
return 0;
}

int
pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id)
{
return 0;
}

// pthread
int
pthread_create(pthread_t *restrict thread,
const pthread_attr_t *restrict attr,
void *(*start_routine)(void *),
void *restrict arg)
{
return EAGAIN;
}

int
pthread_detach(pthread_t thread)
{
return 0;
}

PyAPI_FUNC(pthread_t) pthread_self(void)
{
return 0;
}

int
pthread_exit(void *retval)
{
exit(0);
}

int
pthread_attr_init(pthread_attr_t *attr)
{
return 0;
}

int
pthread_attr_setstacksize(
pthread_attr_t *attr, size_t stacksize)
{
return 0;
}

int
pthread_attr_destroy(pthread_attr_t *attr)
{
return 0;
}

// pthread_key
typedef struct {
bool in_use;
void *value;
} py_tls_entry;

static py_tls_entry py_tls_entries[PTHREAD_KEYS_MAX] = {0};

int
pthread_key_create(pthread_key_t *key, void (*destr_function)(void *))
{
if (!key) {
return EINVAL;
}
if (destr_function != NULL) {
Py_FatalError("pthread_key_create destructor is not supported");
}
for (pthread_key_t idx = 0; idx < PTHREAD_KEYS_MAX; idx++) {
if (!py_tls_entries[idx].in_use) {
py_tls_entries[idx].in_use = true;
*key = idx;
return 0;
}
}
return EAGAIN;
}

int
pthread_key_delete(pthread_key_t key)
{
if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
return EINVAL;
}
py_tls_entries[key].in_use = false;
py_tls_entries[key].value = NULL;
return 0;
}


void *
pthread_getspecific(pthread_key_t key) {
if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
return NULL;
}
return py_tls_entries[key].value;
}

int
pthread_setspecific(pthread_key_t key, const void *value)
{
if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
return EINVAL;
}
py_tls_entries[key].value = (void *)value;
return 0;
}

// let thread_pthread define the Python API
#include "thread_pthread.h"
14 changes: 3 additions & 11 deletions Tools/wasm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -226,9 +226,7 @@ AddType application/wasm wasm

# WASI (wasm32-wasi)

WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 15.0+
and currently [wasix](https://github.com/singlestore-labs/wasix) for POSIX
compatibility stubs.
WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+.

## Cross-compile to wasm32-wasi

Expand Down Expand Up @@ -434,21 +432,15 @@ rm -f wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz

### Install [wasmtime](https://github.com/bytecodealliance/wasmtime) WASI runtime

**NOTE**: wasmtime 0.37 has a bug. Newer versions should be fine again.
wasmtime 0.38 or newer is required.

```shell
curl -sSf -L -o ~/install-wasmtime.sh https://wasmtime.dev/install.sh
chmod +x ~/install-wasmtime.sh
~/install-wasmtime.sh --version v0.36.0
~/install-wasmtime.sh --version v0.38.0
ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime
```

### Install [WASIX](https://github.com/singlestore-labs/wasix)

```shell
git clone https://github.com/singlestore-labs/wasix.git ~/wasix
make install -C ~/wasix
```

### WASI debugging

Expand Down
Loading

0 comments on commit d707f52

Please sign in to comment.