From da0e917ed3cf2766c33b13b1edaed770805af529 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 22 May 2024 22:55:54 +0200 Subject: [PATCH 01/17] Make the _json module thread safe --- Modules/_json.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Modules/_json.c b/Modules/_json.c index e33ef1f5eea92f..8ef69420e305c4 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_runtime.h" // _PyRuntime #include "pycore_global_strings.h" // _Py_ID() @@ -1615,12 +1616,14 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; + Py_BEGIN_CRITICAL_SECTION(dct); while (PyDict_Next(dct, &pos, &key, &value)) { if (encoder_encode_key_value(s, writer, &first, key, value, new_newline_indent, current_item_separator) < 0) goto bail; } + Py_END_CRITICAL_SECTION(dct); } if (ident != NULL) { @@ -1661,6 +1664,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); + if (s_fast == NULL) return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { @@ -1713,6 +1718,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) goto bail; } + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) goto bail; From 3797dfac3cf046267b05df9e022e5946ebe6c914 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 23 May 2024 00:10:44 +0200 Subject: [PATCH 02/17] Update Modules/_json.c --- Modules/_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_json.c b/Modules/_json.c index 8ef69420e305c4..16ab12c80384d1 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1623,7 +1623,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, current_item_separator) < 0) goto bail; } - Py_END_CRITICAL_SECTION(dct); + Py_END_CRITICAL_SECTION(); } if (ident != NULL) { From 366654c196d928596023ffd72a7d6497ea7c83f0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 24 May 2024 22:13:23 +0200 Subject: [PATCH 03/17] handle goto and return statements --- Include/internal/pycore_critical_section.h | 7 +++++++ Modules/_json.c | 14 ++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 7bebcdb67c7169..ce1640bd7a5a65 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -128,6 +128,12 @@ extern "C" { } \ } +# define Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(value) \ + if (_should_lock_cs) { \ + _PyCriticalSection_End(&_cs); \ + } \ + return value; + // Asserts that the mutex is locked. The mutex must be held by the // top-most critical section otherwise there's the possibility // that the mutex would be swalled out in some code paths. @@ -159,6 +165,7 @@ extern "C" { # define Py_END_CRITICAL_SECTION2() # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() +# define Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(value) # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ diff --git a/Modules/_json.c b/Modules/_json.c index 16ab12c80384d1..f53756be775129 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1664,13 +1664,15 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); - Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); - if (s_fast == NULL) return -1; + + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); + if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); + int r = _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); + Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(r); } if (s->markers != Py_None) { @@ -1718,13 +1720,11 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) goto bail; } - Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) goto bail; Py_CLEAR(ident); } - if (s->indent != Py_None) { Py_CLEAR(new_newline_indent); Py_CLEAR(separator_indent); @@ -1736,14 +1736,16 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, ']')) goto bail; Py_DECREF(s_fast); - return 0; + Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(0); bail: + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); Py_XDECREF(ident); Py_DECREF(s_fast); Py_XDECREF(separator_indent); Py_XDECREF(new_newline_indent); return -1; + } static void From 5b72cdf3214357cd8908b30a21d59e4fa408da35 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 25 May 2024 12:45:41 +0200 Subject: [PATCH 04/17] Apply suggestions from code review Co-authored-by: Nice Zombies --- Modules/_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_json.c b/Modules/_json.c index f53756be775129..a1f5c45df77f33 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1725,6 +1725,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } + if (s->indent != Py_None) { Py_CLEAR(new_newline_indent); Py_CLEAR(separator_indent); @@ -1745,7 +1746,6 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_XDECREF(separator_indent); Py_XDECREF(new_newline_indent); return -1; - } static void From c4c24c38a3eabafe929b3dfdb3c602bd10dd11a0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 25 May 2024 12:47:17 +0200 Subject: [PATCH 05/17] Update Include/internal/pycore_critical_section.h --- Include/internal/pycore_critical_section.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index ce1640bd7a5a65..91e0d6a15ee38f 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -165,7 +165,7 @@ extern "C" { # define Py_END_CRITICAL_SECTION2() # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() -# define Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(value) +# define Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(value) return value; # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ From 370191b9335a93a618a491de3d89e56ae2512e4d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 31 May 2024 23:31:09 +0200 Subject: [PATCH 06/17] rename macro --- Include/internal/pycore_critical_section.h | 7 +++---- Modules/_json.c | 7 ++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 91e0d6a15ee38f..8cb75a019fa13b 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -128,11 +128,10 @@ extern "C" { } \ } -# define Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(value) \ +# define Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST \ if (_should_lock_cs) { \ _PyCriticalSection_End(&_cs); \ - } \ - return value; + } // Asserts that the mutex is locked. The mutex must be held by the // top-most critical section otherwise there's the possibility @@ -165,7 +164,7 @@ extern "C" { # define Py_END_CRITICAL_SECTION2() # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() -# define Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(value) return value; +# define Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST() # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ diff --git a/Modules/_json.c b/Modules/_json.c index a1f5c45df77f33..46adffa5a1e60d 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1671,8 +1671,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - int r = _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); - Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(r); + Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST(); + return _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); } if (s->markers != Py_None) { @@ -1737,7 +1737,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, ']')) goto bail; Py_DECREF(s_fast); - Py_RETURN_CRITICAL_SECTION_SEQUENCE_FAST(0); + Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST(); + return 0; bail: Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); From eafd3c1cd92b4e5abfd8e013584de6e9303a47b1 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 31 May 2024 23:34:34 +0200 Subject: [PATCH 07/17] fix typo --- Include/internal/pycore_critical_section.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 8cb75a019fa13b..a11af1bf0c0d8f 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -128,7 +128,7 @@ extern "C" { } \ } -# define Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST \ +# define Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST() \ if (_should_lock_cs) { \ _PyCriticalSection_End(&_cs); \ } From d54baf2049a71956b26c45e67fe31bf4601517e8 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 4 Jun 2024 15:26:06 +0200 Subject: [PATCH 08/17] fix missing to exit critical section --- Include/internal/pycore_critical_section.h | 4 ++++ Lib/test/test_json/test_dump.py | 4 +++- Modules/_json.c | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index a11af1bf0c0d8f..adebcae7e24076 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -99,6 +99,9 @@ extern "C" { _PyCriticalSection_End(&_cs); \ } +# define Py_EXIT_CRITICAL_SECTION() \ + _PyCriticalSection_End(&_cs); + # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { \ _PyCriticalSection2 _cs2; \ @@ -160,6 +163,7 @@ extern "C" { # define Py_BEGIN_CRITICAL_SECTION_MUT(mut) # define Py_BEGIN_CRITICAL_SECTION(op) # define Py_END_CRITICAL_SECTION() +# define Py_EXIT_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 13b40020781bae..fc052dfa37afb9 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -12,11 +12,13 @@ def test_dump(self): def test_dumps(self): self.assertEqual(self.dumps({}), '{}') - def test_dump_skipkeys(self): + def test_dump_skipkeys_invalid(self): v = {b'invalid_key': False, 'valid_key': True} with self.assertRaises(TypeError): self.json.dumps(v) + def test_dump_skipkeys(self): + v = {b'invalid_key': False, 'valid_key': True} s = self.json.dumps(v, skipkeys=True) o = self.json.loads(s) self.assertIn('valid_key', o) diff --git a/Modules/_json.c b/Modules/_json.c index d2b308eca36d3f..be8224e042d18a 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1620,8 +1620,10 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, while (PyDict_Next(dct, &pos, &key, &value)) { if (encoder_encode_key_value(s, writer, &first, key, value, new_newline_indent, - current_item_separator) < 0) + current_item_separator) < 0) { + Py_EXIT_CRITICAL_SECTION(); goto bail; + } } Py_END_CRITICAL_SECTION(); } From e5fa305ad7842d71a552b9d97431e7b56a2604fb Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 4 Jun 2024 22:25:59 +0200 Subject: [PATCH 09/17] revert changes to tests --- Lib/test/test_json/test_dump.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index fc052dfa37afb9..13b40020781bae 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -12,13 +12,11 @@ def test_dump(self): def test_dumps(self): self.assertEqual(self.dumps({}), '{}') - def test_dump_skipkeys_invalid(self): + def test_dump_skipkeys(self): v = {b'invalid_key': False, 'valid_key': True} with self.assertRaises(TypeError): self.json.dumps(v) - def test_dump_skipkeys(self): - v = {b'invalid_key': False, 'valid_key': True} s = self.json.dumps(v, skipkeys=True) o = self.json.loads(s) self.assertIn('valid_key', o) From d4ddf5dcab9f8f0b9bc5eede9f9d96f0b07d1df9 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 20:26:25 +0000 Subject: [PATCH 10/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst new file mode 100644 index 00000000000000..d7d64b8a8edc53 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst @@ -0,0 +1 @@ +Make the module :mod:`json` thread-safe. From 384ca59bf6ed251f8681c8ba9e0a6607a614c678 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 14 Aug 2024 22:37:12 +0200 Subject: [PATCH 11/17] sync with main --- Include/cpython/critical_section.h | 4 ++++ Include/internal/pycore_critical_section.h | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 35db3fb6a59ce6..40862e90f22144 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -118,6 +118,10 @@ struct PyCriticalSection2 { PyCriticalSection _py_cs; \ PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +# define Py_EXIT_CRITICAL_SECTION() \ + _PyCriticalSection_End(&_py_cs); + + # define Py_END_CRITICAL_SECTION() \ PyCriticalSection_End(&_py_cs); \ } diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 1e8440469dd404..76b18f4519c653 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -26,9 +26,6 @@ extern "C" { PyCriticalSection _py_cs; \ _PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_EXIT_CRITICAL_SECTION() \ - _PyCriticalSection_End(&_cs); - # define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ { \ PyCriticalSection2 _py_cs2; \ From 64e20aa80bffb98437775f8c21ec2299adf2b806 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 14 Aug 2024 22:41:51 +0200 Subject: [PATCH 12/17] sync with main --- Include/cpython/critical_section.h | 1 + Include/internal/pycore_critical_section.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 40862e90f22144..d4c3ec9c67e504 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -85,6 +85,7 @@ PyCriticalSection2_End(PyCriticalSection2 *c); #ifndef Py_GIL_DISABLED # define Py_BEGIN_CRITICAL_SECTION(op) \ { +# define Py_EXIT_CRITICAL_SECTION() # define Py_END_CRITICAL_SECTION() \ } # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 76b18f4519c653..f19780754edcc1 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -83,7 +83,6 @@ extern "C" { # define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { # define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { -# define Py_EXIT_CRITICAL_SECTION() # define Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST() # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) From e6ce9c9936e71fefe351a6a2f0e8544216cfca9d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 14 Aug 2024 22:43:32 +0200 Subject: [PATCH 13/17] update news entry --- .../2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst index d7d64b8a8edc53..8ae38e757ff925 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-04-20-26-21.gh-issue-116738.q_hPYq.rst @@ -1 +1 @@ -Make the module :mod:`json` thread-safe. +Make the module :mod:`json` safe to use under the free-theading build. From 34885a02f5dad38b7e3f98393f6cd382ef6b29dd Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 14 Aug 2024 22:50:17 +0200 Subject: [PATCH 14/17] fix normal build --- Modules/_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_json.c b/Modules/_json.c index 5d6c635cc40f51..9fa5d1182c3073 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1748,8 +1748,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, return 0; bail: - Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); Py_XDECREF(ident); + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); Py_DECREF(s_fast); Py_XDECREF(separator_indent); Py_XDECREF(new_newline_indent); From eebccac0f005a83ed459af42a5a3c0f87d70ef02 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 15 Aug 2024 17:42:13 +0200 Subject: [PATCH 15/17] add lock around result of PyMapping_Items --- Modules/_json.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_json.c b/Modules/_json.c index 9fa5d1182c3073..93c207eac1406e 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1600,11 +1600,13 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) goto bail; + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(items); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { PyObject *item = PyList_GET_ITEM(items, i); if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); + Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST(); goto bail; } @@ -1612,9 +1614,12 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, value = PyTuple_GET_ITEM(item, 1); if (encoder_encode_key_value(s, writer, &first, dct, key, value, new_newline_indent, - current_item_separator) < 0) + current_item_separator) < 0) { + Py_EXIT_CRITICAL_SECTION_SEQUENCE_FAST(); goto bail; + } } + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(items); Py_CLEAR(items); } else { From db8947cef5acf1fd5751ac5f5e932e43a464e872 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 15 Aug 2024 20:44:03 +0200 Subject: [PATCH 16/17] add tests --- Lib/test/test_json/test_threading.py | 80 ++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Lib/test/test_json/test_threading.py diff --git a/Lib/test/test_json/test_threading.py b/Lib/test/test_json/test_threading.py new file mode 100644 index 00000000000000..ab07cceceda289 --- /dev/null +++ b/Lib/test/test_json/test_threading.py @@ -0,0 +1,80 @@ +import unittest +from threading import Thread +from test.test_json import CTest +from test.support import threading_helper + + +def encode_json_helper(json, worker, data, number_of_threads, number_of_json_encodings=100): + worker_threads = [] + for index in range(number_of_threads): + worker_threads.append(Thread(target=worker, args=[data, index])) + for t in worker_threads: + t.start() + for ii in range(number_of_json_encodings): + json.dumps(data) + data.clear() + for t in worker_threads: + t.join() + + +class MyMapping(dict): + def __init__(self): + self.mapping = [] + + def items(self): + return self.mapping + + +@threading_helper.reap_threads +@threading_helper.requires_working_threading() +class TestJsonEncoding(CTest): + # Test encoding json with multiple threads modifying the data cannot + # corrupt the interpreter + + def test_json_mutating_list(self): + + def worker(data, index): + while data: + for d in data: + if len(d) > 5: + d.clear() + else: + d.append(index) + d.append(index) + d.append(index) + encode_json_helper(self.json, worker, [[], []], number_of_threads=16) + + def test_json_mutating_dict(self): + + def worker(data, index): + while data: + for d in data: + if len(d) > 5: + try: + d.pop(list(d)[0]) + except (KeyError, IndexError): + pass + else: + d[index] = index + encode_json_helper(self.json, worker, [{}, {}], number_of_threads=16) + + def test_json_mutating_mapping(self): + + def worker(data, index): + while data: + for d in data: + if len(d.mapping) > 3: + d.mapping.clear() + else: + d.mapping.append((index, index)) + encode_json_helper(self.json, + worker, [MyMapping(), MyMapping()], number_of_threads=16) + + +if __name__ == "__main__": + import time + + t0 = time.time() + unittest.main() + dt = time.time()-t0 + print(f'Done: {dt:.2f}') From c19ad14886ac198a8329960bd1732d71c6bcd077 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 15 Aug 2024 20:55:38 +0200 Subject: [PATCH 17/17] fix argument of Py_END_CRITICAL_SECTION_SEQUENCE_FAST --- Modules/_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_json.c b/Modules/_json.c index 93c207eac1406e..49b688d28b5c41 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1619,7 +1619,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; } } - Py_END_CRITICAL_SECTION_SEQUENCE_FAST(items); + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); Py_CLEAR(items); } else {