diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index b365ca337eabc8..9900ce98037060 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -35,7 +35,7 @@ extern "C" { struct _Py_list_freelist { #ifdef WITH_FREELISTS - PyListObject *free_list[PyList_MAXFREELIST]; + PyListObject *items[PyList_MAXFREELIST]; int numfree; #endif }; @@ -50,7 +50,7 @@ struct _Py_tuple_freelist { object is used as the linked list node, with its first item (ob_item[0]) pointing to the next node (i.e. the previous head). Each linked list is initially NULL. */ - PyTupleObject *free_list[PyTuple_NFREELISTS]; + PyTupleObject *items[PyTuple_NFREELISTS]; int numfree[PyTuple_NFREELISTS]; #else char _unused; // Empty structs are not allowed. @@ -63,17 +63,23 @@ struct _Py_float_freelist { free_list is a singly-linked list of available PyFloatObjects, linked via abuse of their ob_type members. */ int numfree; - PyFloatObject *free_list; + PyFloatObject *items; #endif }; struct _Py_dict_freelist { #ifdef WITH_FREELISTS /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; + PyDictObject *items[PyDict_MAXFREELIST]; + int numfree; +#endif +}; + +struct _Py_dictkeys_freelist { +#ifdef WITH_FREELISTS + /* Dictionary keys reuse scheme to save calls to malloc and free */ + PyDictKeysObject *items[PyDict_MAXFREELIST]; int numfree; - int keys_numfree; #endif }; @@ -88,7 +94,7 @@ struct _Py_slice_freelist { struct _Py_context_freelist { #ifdef WITH_FREELISTS // List of free PyContext objects - PyContext *freelist; + PyContext *items; int numfree; #endif }; @@ -110,7 +116,7 @@ struct _Py_async_gen_freelist { struct _PyObjectStackChunk; struct _Py_object_stack_freelist { - struct _PyObjectStackChunk *free_list; + struct _PyObjectStackChunk *items; Py_ssize_t numfree; }; @@ -119,6 +125,7 @@ struct _Py_object_freelists { struct _Py_tuple_freelist tuples; struct _Py_list_freelist lists; struct _Py_dict_freelist dicts; + struct _Py_dictkeys_freelist dictkeys; struct _Py_slice_freelist slices; struct _Py_context_freelist contexts; struct _Py_async_gen_freelist async_gens; diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 664f6fb212a57d..9134ab45cd0039 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -22,7 +22,6 @@ struct _types_runtime_state { // bpo-42745: next_version_tag remains shared by all interpreters // because of static types. unsigned int next_version_tag; - PyMutex type_mutex; }; @@ -71,6 +70,7 @@ struct types_state { struct type_cache type_cache; size_t num_builtins_initialized; static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; + PyMutex mutex; }; diff --git a/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst b/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst new file mode 100644 index 00000000000000..b4f9fe64db0615 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst @@ -0,0 +1,4 @@ +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index bc302909424227..950ee3663080e1 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4553,6 +4553,50 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c) return 0; } +#if OPENSSL_VERSION_NUMBER < 0x30300000L +static X509_OBJECT *x509_object_dup(const X509_OBJECT *obj) +{ + int ok; + X509_OBJECT *ret = X509_OBJECT_new(); + if (ret == NULL) { + return NULL; + } + switch (X509_OBJECT_get_type(obj)) { + case X509_LU_X509: + ok = X509_OBJECT_set1_X509(ret, X509_OBJECT_get0_X509(obj)); + break; + case X509_LU_CRL: + /* X509_OBJECT_get0_X509_CRL was not const-correct prior to 3.0.*/ + ok = X509_OBJECT_set1_X509_CRL( + ret, X509_OBJECT_get0_X509_CRL((X509_OBJECT *)obj)); + break; + default: + /* We cannot duplicate unrecognized types in a polyfill, but it is + * safe to leave an empty object. The caller will ignore it. */ + ok = 1; + break; + } + if (!ok) { + X509_OBJECT_free(ret); + return NULL; + } + return ret; +} + +static STACK_OF(X509_OBJECT) * +X509_STORE_get1_objects(X509_STORE *store) +{ + STACK_OF(X509_OBJECT) *ret; + if (!X509_STORE_lock(store)) { + return NULL; + } + ret = sk_X509_OBJECT_deep_copy(X509_STORE_get0_objects(store), + x509_object_dup, X509_OBJECT_free); + X509_STORE_unlock(store); + return ret; +} +#endif + PyDoc_STRVAR(PySSLContext_sni_callback_doc, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\ \n\ @@ -4582,7 +4626,12 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) int x509 = 0, crl = 0, ca = 0, i; store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + return NULL; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { obj = sk_X509_OBJECT_value(objs, i); switch (X509_OBJECT_get_type(obj)) { @@ -4596,12 +4645,11 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) crl++; break; default: - /* Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY. - * As far as I can tell they are internal states and never - * stored in a cert store */ + /* Ignore unrecognized types. */ break; } } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return Py_BuildValue("{sisisi}", "x509", x509, "crl", crl, "x509_ca", ca); } @@ -4633,7 +4681,12 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } store = SSL_CTX_get_cert_store(self->ctx); - objs = X509_STORE_get0_objects(store); + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + goto error; + } + for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { X509_OBJECT *obj; X509 *cert; @@ -4661,9 +4714,11 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } Py_CLEAR(ci); } + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); return rlist; error: + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); Py_XDECREF(ci); Py_XDECREF(rlist); return NULL; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 11667b07ecfb4b..25ab21881f8f74 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -276,6 +276,13 @@ get_dict_freelist(void) struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); return &freelists->dicts; } + +static struct _Py_dictkeys_freelist * +get_dictkeys_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->dictkeys; +} #endif @@ -283,18 +290,19 @@ void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_dict_freelist *state = &freelists->dicts; - while (state->numfree > 0) { - PyDictObject *op = state->free_list[--state->numfree]; + struct _Py_dict_freelist *freelist = &freelists->dicts; + while (freelist->numfree > 0) { + PyDictObject *op = freelist->items[--freelist->numfree]; assert(PyDict_CheckExact(op)); PyObject_GC_Del(op); } - while (state->keys_numfree > 0) { - PyMem_Free(state->keys_free_list[--state->keys_numfree]); + struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys; + while (keys_freelist->numfree > 0) { + PyMem_Free(keys_freelist->items[--keys_freelist->numfree]); } if (is_finalization) { - state->numfree = -1; - state->keys_numfree = -1; + freelist->numfree = -1; + keys_freelist->numfree = -1; } #endif } @@ -314,6 +322,9 @@ _PyDict_DebugMallocStats(FILE *out) struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); _PyDebugAllocatorStats(out, "free PyDictObject", dict_freelist->numfree, sizeof(PyDictObject)); + struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); + _PyDebugAllocatorStats(out, "free PyDictKeysObject", + dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); #endif } @@ -663,9 +674,9 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } #ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); - if (log2_size == PyDict_LOG_MINSIZE && unicode && dict_freelist->keys_numfree > 0) { - dk = dict_freelist->keys_free_list[--dict_freelist->keys_numfree]; + struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); + if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { + dk = freelist->items[--freelist->numfree]; OBJECT_STAT_INC(from_freelist); } else @@ -698,12 +709,12 @@ static void free_keys_object(PyDictKeysObject *keys) { #ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); + struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && dict_freelist->keys_numfree < PyDict_MAXFREELIST - && dict_freelist->keys_numfree >= 0 + && freelist->numfree < PyDict_MAXFREELIST + && freelist->numfree >= 0 && DK_IS_UNICODE(keys)) { - dict_freelist->keys_free_list[dict_freelist->keys_numfree++] = keys; + freelist->items[freelist->numfree++] = keys; OBJECT_STAT_INC(to_freelist); return; } @@ -743,9 +754,9 @@ new_dict(PyInterpreterState *interp, PyDictObject *mp; assert(keys != NULL); #ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); - if (dict_freelist->numfree > 0) { - mp = dict_freelist->free_list[--dict_freelist->numfree]; + struct _Py_dict_freelist *freelist = get_dict_freelist(); + if (freelist->numfree > 0) { + mp = freelist->items[--freelist->numfree]; assert (mp != NULL); assert (Py_IS_TYPE(mp, &PyDict_Type)); OBJECT_STAT_INC(from_freelist); @@ -2593,10 +2604,10 @@ dict_dealloc(PyObject *self) dictkeys_decref(interp, keys); } #ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); - if (dict_freelist->numfree < PyDict_MAXFREELIST && dict_freelist->numfree >=0 && + struct _Py_dict_freelist *freelist = get_dict_freelist(); + if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 && Py_IS_TYPE(mp, &PyDict_Type)) { - dict_freelist->free_list[dict_freelist->numfree++] = mp; + freelist->items[freelist->numfree++] = mp; OBJECT_STAT_INC(to_freelist); } else diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 7dac8292c7232b..37d2d312a6a0b7 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -130,9 +130,9 @@ PyFloat_FromDouble(double fval) PyFloatObject *op; #ifdef WITH_FREELISTS struct _Py_float_freelist *float_freelist = get_float_freelist(); - op = float_freelist->free_list; + op = float_freelist->items; if (op != NULL) { - float_freelist->free_list = (PyFloatObject *) Py_TYPE(op); + float_freelist->items = (PyFloatObject *) Py_TYPE(op); float_freelist->numfree--; OBJECT_STAT_INC(from_freelist); } @@ -251,8 +251,8 @@ _PyFloat_ExactDealloc(PyObject *obj) return; } float_freelist->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)float_freelist->free_list); - float_freelist->free_list = op; + Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items); + float_freelist->items = op; OBJECT_STAT_INC(to_freelist); #else PyObject_Free(op); @@ -1994,13 +1994,13 @@ _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalizati { #ifdef WITH_FREELISTS struct _Py_float_freelist *state = &freelists->floats; - PyFloatObject *f = state->free_list; + PyFloatObject *f = state->items; while (f != NULL) { PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); PyObject_Free(f); f = next; } - state->free_list = NULL; + state->items = NULL; if (is_finalization) { state->numfree = -1; } diff --git a/Objects/listobject.c b/Objects/listobject.c index 96182a42306d95..eb466260318ec1 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -133,7 +133,7 @@ _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalizatio #ifdef WITH_FREELISTS struct _Py_list_freelist *state = &freelists->lists; while (state->numfree > 0) { - PyListObject *op = state->free_list[--state->numfree]; + PyListObject *op = state->items[--state->numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } @@ -169,7 +169,7 @@ PyList_New(Py_ssize_t size) struct _Py_list_freelist *list_freelist = get_list_freelist(); if (PyList_MAXFREELIST && list_freelist->numfree > 0) { list_freelist->numfree--; - op = list_freelist->free_list[list_freelist->numfree]; + op = list_freelist->items[list_freelist->numfree]; OBJECT_STAT_INC(from_freelist); _Py_NewReference((PyObject *)op); } @@ -401,7 +401,7 @@ list_dealloc(PyObject *self) #ifdef WITH_FREELISTS struct _Py_list_freelist *list_freelist = get_list_freelist(); if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) { - list_freelist->free_list[list_freelist->numfree++] = op; + list_freelist->items[list_freelist->numfree++] = op; OBJECT_STAT_INC(to_freelist); } else diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 1cdf79d95ae352..d9dc00da368a84 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1134,11 +1134,11 @@ maybe_freelist_pop(Py_ssize_t size) assert(size > 0); if (size < PyTuple_MAXSAVESIZE) { Py_ssize_t index = size - 1; - PyTupleObject *op = TUPLE_FREELIST.free_list[index]; + PyTupleObject *op = TUPLE_FREELIST.items[index]; if (op != NULL) { /* op is the head of a linked list, with the first item pointing to the next node. Here we pop off the old head. */ - TUPLE_FREELIST.free_list[index] = (PyTupleObject *) op->ob_item[0]; + TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0]; TUPLE_FREELIST.numfree[index]--; /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ #ifdef Py_TRACE_REFS @@ -1173,8 +1173,8 @@ maybe_freelist_push(PyTupleObject *op) { /* op is the head of a linked list, with the first item pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) TUPLE_FREELIST.free_list[index]; - TUPLE_FREELIST.free_list[index] = op; + op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index]; + TUPLE_FREELIST.items[index] = op; TUPLE_FREELIST.numfree[index]++; OBJECT_STAT_INC(to_freelist); return 1; @@ -1188,8 +1188,8 @@ maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini) { #ifdef WITH_FREELISTS for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = TUPLE_FREELIST.free_list[i]; - TUPLE_FREELIST.free_list[i] = NULL; + PyTupleObject *p = TUPLE_FREELIST.items[i]; + TUPLE_FREELIST.items[i] = NULL; TUPLE_FREELIST.numfree[i] = fini ? -1 : 0; while (p) { PyTupleObject *q = p; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e0711dfe8545b7..0118ee255ef017 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -60,17 +60,18 @@ class object "PyObject *" "&PyBaseObject_Type" // in odd behaviors w.r.t. running with the GIL as the outer type lock could // be released and reacquired during a subclass update if there's contention // on the subclass lock. +#define TYPE_LOCK &PyInterpreterState_Get()->types.mutex #define BEGIN_TYPE_LOCK() \ { \ _PyCriticalSection _cs; \ - _PyCriticalSection_Begin(&_cs, &_PyRuntime.types.type_mutex); \ + _PyCriticalSection_Begin(&_cs, TYPE_LOCK); \ #define END_TYPE_LOCK() \ _PyCriticalSection_End(&_cs); \ } #define ASSERT_TYPE_LOCK_HELD() \ - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyRuntime.types.type_mutex) + _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) #else diff --git a/Python/context.c b/Python/context.c index 01a21b47da5452..3937819b3c386c 100644 --- a/Python/context.c +++ b/Python/context.c @@ -344,8 +344,8 @@ _context_alloc(void) struct _Py_context_freelist *context_freelist = get_context_freelist(); if (context_freelist->numfree > 0) { context_freelist->numfree--; - ctx = context_freelist->freelist; - context_freelist->freelist = (PyContext *)ctx->ctx_weakreflist; + ctx = context_freelist->items; + context_freelist->items = (PyContext *)ctx->ctx_weakreflist; OBJECT_STAT_INC(from_freelist); ctx->ctx_weakreflist = NULL; _Py_NewReference((PyObject *)ctx); @@ -471,8 +471,8 @@ context_tp_dealloc(PyContext *self) struct _Py_context_freelist *context_freelist = get_context_freelist(); if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) { context_freelist->numfree++; - self->ctx_weakreflist = (PyObject *)context_freelist->freelist; - context_freelist->freelist = self; + self->ctx_weakreflist = (PyObject *)context_freelist->items; + context_freelist->items = self; OBJECT_STAT_INC(to_freelist); } else @@ -1272,8 +1272,8 @@ _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finaliza #ifdef WITH_FREELISTS struct _Py_context_freelist *state = &freelists->contexts; for (; state->numfree > 0; state->numfree--) { - PyContext *ctx = state->freelist; - state->freelist = (PyContext *)ctx->ctx_weakreflist; + PyContext *ctx = state->items; + state->items = (PyContext *)ctx->ctx_weakreflist; ctx->ctx_weakreflist = NULL; PyObject_GC_Del(ctx); } diff --git a/Python/object_stack.c b/Python/object_stack.c index ff2901cdacceb8..bd9696822c8a9d 100644 --- a/Python/object_stack.c +++ b/Python/object_stack.c @@ -21,8 +21,8 @@ _PyObjectStackChunk_New(void) _PyObjectStackChunk *buf; struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); if (obj_stack_freelist->numfree > 0) { - buf = obj_stack_freelist->free_list; - obj_stack_freelist->free_list = buf->prev; + buf = obj_stack_freelist->items; + obj_stack_freelist->items = buf->prev; obj_stack_freelist->numfree--; } else { @@ -47,8 +47,8 @@ _PyObjectStackChunk_Free(_PyObjectStackChunk *buf) if (obj_stack_freelist->numfree >= 0 && obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST) { - buf->prev = obj_stack_freelist->free_list; - obj_stack_freelist->free_list = buf; + buf->prev = obj_stack_freelist->items; + obj_stack_freelist->items = buf; obj_stack_freelist->numfree++; } else { @@ -97,12 +97,12 @@ _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is return; } - struct _Py_object_stack_freelist *state = &freelists->object_stacks; - while (state->numfree > 0) { - _PyObjectStackChunk *buf = state->free_list; - state->free_list = buf->prev; - state->numfree--; + struct _Py_object_stack_freelist *freelist = &freelists->object_stacks; + while (freelist->numfree > 0) { + _PyObjectStackChunk *buf = freelist->items; + freelist->items = buf->prev; + freelist->numfree--; PyMem_RawFree(buf); } - state->numfree = -1; + freelist->numfree = -1; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5e5db98481150e..b354c033ae7727 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1109,7 +1109,6 @@ run_presite(PyThreadState *tstate) ); if (presite_modname == NULL) { fprintf(stderr, "Could not convert pre-site module name to unicode\n"); - Py_DECREF(presite_modname); } else { PyObject *presite = PyImport_Import(presite_modname); diff --git a/Python/pystate.c b/Python/pystate.c index b1d1a085d629b4..24f9b7790915ab 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -395,7 +395,7 @@ _Py_COMP_DIAG_POP &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ - &(runtime)->types.type_mutex, \ + &(runtime)->_main_interpreter.types.mutex, \ } static void diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 6237809764d9e1..738864a48c08d3 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -16,6 +16,11 @@ wrap_declarations, wrapped_c_string_literal, ) +from .identifiers import ( + ensure_legal_c_identifier, + is_legal_c_identifier, + is_legal_py_identifier, +) from .utils import ( FormatCounterFormatter, compute_checksum, @@ -41,6 +46,11 @@ "wrap_declarations", "wrapped_c_string_literal", + # Identifier helpers + "ensure_legal_c_identifier", + "is_legal_c_identifier", + "is_legal_py_identifier", + # Utility functions "FormatCounterFormatter", "compute_checksum", diff --git a/Tools/clinic/libclinic/clinic.py b/Tools/clinic/libclinic/clinic.py index c35df2f39c32fc..2559a0dad63353 100644 --- a/Tools/clinic/libclinic/clinic.py +++ b/Tools/clinic/libclinic/clinic.py @@ -134,31 +134,6 @@ def fail( warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) -is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match - -def is_legal_py_identifier(s: str) -> bool: - return all(is_legal_c_identifier(field) for field in s.split('.')) - -# identifiers that are okay in Python but aren't a good idea in C. -# so if they're used Argument Clinic will add "_value" to the end -# of the name in C. -c_keywords = set(""" -asm auto break case char const continue default do double -else enum extern float for goto if inline int long -register return short signed sizeof static struct switch -typedef typeof union unsigned void volatile while -""".strip().split()) - -def ensure_legal_c_identifier(s: str) -> str: - # for now, just complain if what we're given isn't legal - if not is_legal_c_identifier(s): - fail("Illegal C identifier:", s) - # but if we picked a C keyword, pick something else - if s in c_keywords: - return s + "_value" - return s - - class CRenderData: def __init__(self) -> None: @@ -2950,7 +2925,7 @@ def __init__(self, unused: bool = False, **kwargs: Any ) -> None: - self.name = ensure_legal_c_identifier(name) + self.name = libclinic.ensure_legal_c_identifier(name) self.py_name = py_name self.unused = unused self.includes: list[Include] = [] @@ -5079,9 +5054,9 @@ def parse_function_names(self, line: str) -> FunctionNames: if fields[-1] == '__new__': fields.pop() c_basename = "_".join(fields) - if not is_legal_py_identifier(full_name): + if not libclinic.is_legal_py_identifier(full_name): fail(f"Illegal function name: {full_name!r}") - if not is_legal_c_identifier(c_basename): + if not libclinic.is_legal_c_identifier(c_basename): fail(f"Illegal C basename: {c_basename!r}") names = FunctionNames(full_name=full_name, c_basename=c_basename) self.normalize_function_kind(names.full_name) @@ -5203,7 +5178,7 @@ def state_modulename_name(self, line: str) -> None: before, equals, existing = line.rpartition('=') if equals: existing = existing.strip() - if is_legal_py_identifier(existing): + if libclinic.is_legal_py_identifier(existing): # we're cloning! names = self.parse_function_names(before) return self.parse_cloned_function(names, existing) diff --git a/Tools/clinic/libclinic/identifiers.py b/Tools/clinic/libclinic/identifiers.py new file mode 100644 index 00000000000000..d3b80bbcef3b2b --- /dev/null +++ b/Tools/clinic/libclinic/identifiers.py @@ -0,0 +1,31 @@ +import re +from .errors import ClinicError + + +is_legal_c_identifier = re.compile("^[A-Za-z_][A-Za-z0-9_]*$").match + + +def is_legal_py_identifier(identifier: str) -> bool: + return all(is_legal_c_identifier(field) for field in identifier.split(".")) + + +# Identifiers that are okay in Python but aren't a good idea in C. +# So if they're used Argument Clinic will add "_value" to the end +# of the name in C. +_c_keywords = frozenset(""" +asm auto break case char const continue default do double +else enum extern float for goto if inline int long +register return short signed sizeof static struct switch +typedef typeof union unsigned void volatile while +""".strip().split() +) + + +def ensure_legal_c_identifier(identifier: str) -> str: + # For now, just complain if what we're given isn't legal. + if not is_legal_c_identifier(identifier): + raise ClinicError(f"Illegal C identifier: {identifier}") + # But if we picked a C keyword, pick something else. + if identifier in _c_keywords: + return identifier + "_value" + return identifier