From a33ca2ad1fcf857817cba505a788e15cf9d6ed0c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 7 Mar 2023 21:27:46 +0000 Subject: [PATCH 01/35] gh-102493: fix normalization in PyErr_SetObject (#102502) Co-authored-by: Jelle Zijlstra --- Lib/test/test_capi/test_exceptions.py | 28 +++++++++++++++++++ ...-03-07-16-56-28.gh-issue-102493.gTXrcD.rst | 1 + Modules/_testcapi/exceptions.c | 15 ++++++++++ Python/errors.c | 16 ++++++++--- 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b543a1a565a56f5..55f131699a25670 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -140,6 +140,34 @@ def test_err_restore(self): self.assertEqual(1, v.args[0]) self.assertIs(tb, v.__traceback__.tb_next) + def test_set_object(self): + + # new exception as obj is not an exception + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, 42) + self.assertEqual(e.exception.args, (42,)) + + # wraps the exception because unrelated types + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, TypeError(1,2,3)) + wrapped = e.exception.args[0] + self.assertIsInstance(wrapped, TypeError) + self.assertEqual(wrapped.args, (1, 2, 3)) + + # is superclass, so does not wrap + with self.assertRaises(PermissionError) as e: + _testcapi.exc_set_object(OSError, PermissionError(24)) + self.assertEqual(e.exception.args, (24,)) + + class Meta(type): + def __subclasscheck__(cls, sub): + 1/0 + + class Broken(Exception, metaclass=Meta): + pass + + with self.assertRaises(ZeroDivisionError) as e: + _testcapi.exc_set_object(Broken, Broken()) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst new file mode 100644 index 000000000000000..4c4e88ca4e7c3cd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst @@ -0,0 +1 @@ +Fix regression in semantics of normalisation in ``PyErr_SetObject``. diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 43b88ccf261d989..a0575213987ffcf 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -78,6 +78,20 @@ make_exception_with_doc(PyObject *self, PyObject *args, PyObject *kwargs) return PyErr_NewExceptionWithDoc(name, doc, base, dict); } +static PyObject * +exc_set_object(PyObject *self, PyObject *args) +{ + PyObject *exc; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) { + return NULL; + } + + PyErr_SetObject(exc, obj); + return NULL; +} + static PyObject * raise_exception(PyObject *self, PyObject *args) { @@ -247,6 +261,7 @@ static PyMethodDef test_methods[] = { PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), METH_VARARGS | METH_KEYWORDS}, + {"exc_set_object", exc_set_object, METH_VARARGS}, {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, {"set_exc_info", test_set_exc_info, METH_VARARGS}, diff --git a/Python/errors.c b/Python/errors.c index f573bed3d63ef04..bbf6d397ce8097c 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -149,9 +149,16 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) exception); return; } - Py_XINCREF(value); /* Normalize the exception */ - if (value == NULL || (PyObject *)Py_TYPE(value) != exception) { + int is_subclass = 0; + if (value != NULL && PyExceptionInstance_Check(value)) { + is_subclass = PyObject_IsSubclass((PyObject *)Py_TYPE(value), exception); + if (is_subclass < 0) { + return; + } + } + Py_XINCREF(value); + if (!is_subclass) { /* We must normalize the value right now */ PyObject *fixed_value; @@ -206,9 +213,10 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) Py_DECREF(exc_value); } } - if (value != NULL && PyExceptionInstance_Check(value)) + assert(value != NULL); + if (PyExceptionInstance_Check(value)) tb = PyException_GetTraceback(value); - _PyErr_Restore(tstate, Py_XNewRef(exception), value, tb); + _PyErr_Restore(tstate, Py_NewRef(Py_TYPE(value)), value, tb); } void From f9774e57d84162ff0cba0b17a3dcdb93dfbce45e Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 7 Mar 2023 22:41:50 +0100 Subject: [PATCH 02/35] Python 3.12.0a6 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 95 +- Misc/NEWS.d/3.12.0a6.rst | 821 ++++++++++++++++++ ...2-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst | 3 - ...-12-18-08-33-28.gh-issue-100221.K94Ct3.rst | 2 - ...2-12-20-01-06-17.gh-issue-99942.lbmzYj.rst | 2 - ...3-01-12-00-49-16.gh-issue-99942.DUR8b4.rst | 2 - ...2-04-21-17-25-22.gh-issue-91744.FgvaMi.rst | 3 - ...-02-06-16-14-30.gh-issue-101578.PW5fA9.rst | 10 - ...3-02-09-10-38-20.gh-issue-99293.mFqfpp.rst | 2 - ...-02-14-15-53-01.gh-issue-101907.HgF1N2.rst | 1 - ...2-11-02-20-23-47.gh-issue-98627.VJkdRM.rst | 5 - ...-01-04-12-49-33.gh-issue-100719.uRPccL.rst | 3 - ...-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst | 1 - ...-02-08-17-13-31.gh-issue-101696.seJhTt.rst | 1 - ...-02-10-01-15-57.gh-issue-101430.T3Gegb.rst | 2 - ...-02-10-07-21-47.gh-issue-101765.MO5LlC.rst | 1 - ...3-02-10-15-54-57.gh-issue-87849.IUVvPz.rst | 3 - ...3-02-11-23-14-06.gh-issue-84783._P5sMa.rst | 1 - ...-02-12-22-40-22.gh-issue-101857._bribG.rst | 1 - ...-02-13-18-21-14.gh-issue-101799.wpHbCn.rst | 2 - ...3-02-13-22-21-58.gh-issue-74895.esMNtq.rst | 5 - ...-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst | 1 - ...-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst | 1 - ...-02-17-10-12-13.gh-issue-100982.mJGJQw.rst | 2 - ...-02-20-15-18-33.gh-issue-102056.uHKuwH.rst | 1 - ...-02-22-15-15-32.gh-issue-102027.Km4G-d.rst | 2 - ...-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst | 1 - ...-02-26-23-10-32.gh-issue-102250.7MUKoC.rst | 1 - ...-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst | 1 - ...-03-04-20-56-12.gh-issue-102356.07KvUd.rst | 2 - ...-03-06-13-05-33.gh-issue-102416.dz6K5f.rst | 1 - ...-03-07-16-56-28.gh-issue-102493.gTXrcD.rst | 1 - ...3-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst | 2 - ...3-02-19-10-33-01.gh-issue-85417.kYO8u3.rst | 1 - .../2018-06-20-09-12-21.bpo-23224.zxCQ13.rst | 6 - ...2-09-05-12-17-34.gh-issue-88233.gff9qJ.rst | 2 - ...2-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst | 1 - ...3-01-02-22-41-44.gh-issue-99138.17hp9U.rst | 1 - ...-01-06-21-14-41.gh-issue-100809.I697UT.rst | 3 - ...-01-25-00-14-52.gh-issue-101277.FceHX7.rst | 2 - ...-01-27-02-53-50.gh-issue-101360.bPB7SL.rst | 3 - ...3-02-01-10-42-16.gh-issue-63301.XNxSFh.rst | 1 - ...-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst | 1 - ...3-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst | 4 - ...-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst | 2 - ...-02-07-21-16-41.gh-issue-101362.KMQllM.rst | 2 - ...-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst | 4 - ...-02-07-22-21-46.gh-issue-101446.-c0FdK.rst | 2 - ...-02-08-18-20-58.gh-issue-101693.4_LPXj.rst | 6 - ...-02-10-11-59-13.gh-issue-101773.J_kI7y.rst | 2 - ...-02-10-16-02-29.gh-issue-101517.r7S2u8.rst | 1 - ...3-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst | 2 - ...3-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst | 1 - ...-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 3 - ...3-02-15-01-54-06.gh-issue-99108.rjTSic.rst | 3 - ...-02-17-18-44-27.gh-issue-101997.A6_blD.rst | 1 - ...3-02-17-19-00-58.gh-issue-97930.C_nQjb.rst | 4 - ...-02-17-20-24-15.gh-issue-101566.FjgWBt.rst | 4 - ...-02-21-07-15-41.gh-issue-101936.QVOxHH.rst | 2 - ...-02-21-10-05-33.gh-issue-101961.7e56jh.rst | 2 - ...-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst | 1 - ...3-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst | 2 - ...3-02-26-12-37-17.gh-issue-91038.S4rFH_.rst | 1 - ...-02-28-09-52-25.gh-issue-101979.or3hXV.rst | 2 - ...-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst | 1 - ...-01-24-16-12-00.gh-issue-101283.9tqu39.rst | 3 - ...3-02-08-12-57-35.gh-issue-99108.6tnmhA.rst | 4 - ...-02-08-22-03-04.gh-issue-101727.9P5eZz.rst | 4 - ...3-02-17-10-42-48.gh-issue-99108.MKA8-f.rst | 2 - ...3-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 4 - ...3-02-11-22-36-10.gh-issue-85984.EVXjT9.rst | 1 - ...-02-18-10-51-02.gh-issue-102019.0797SJ.rst | 2 - ...-01-25-11-33-54.gh-issue-101196.wAX_2g.rst | 3 - ...-02-07-18-22-54.gh-issue-101614.NjVP0n.rst | 1 - ...-02-09-22-09-27.gh-issue-101759.zFlqSH.rst | 1 - ...-02-10-14-26-05.gh-issue-101763.RPaj7r.rst | 1 - ...-02-13-16-32-50.gh-issue-101849.7lm_53.rst | 1 - ...-02-13-18-05-49.gh-issue-101881._TnHzN.rst | 1 - ...-02-15-11-08-10.gh-issue-101881.fScr3m.rst | 1 - ...-03-01-01-36-39.gh-issue-102344.Dgfux4.rst | 2 - ...-02-09-22-07-17.gh-issue-101759.B0JP2H.rst | 1 - README.rst | 2 +- 83 files changed, 864 insertions(+), 229 deletions(-) create mode 100644 Misc/NEWS.d/3.12.0a6.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst delete mode 100644 Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst delete mode 100644 Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 7957220ed7cf9f0..fd16421eace94af 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 5 +#define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.12.0a5+" +#define PY_VERSION "3.12.0a6" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index e7f403d3ffbf124..573065b4b714d9b 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Feb 7 13:18:04 2023 +# Autogenerated by Sphinx on Tue Mar 7 22:42:28 2023 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -2499,42 +2499,33 @@ 'alive\n' 'until the next garbage collection occurs.\n' '\n' - 'Before an "except" clause’s suite is executed, details about ' - 'the\n' - 'exception are stored in the "sys" module and can be accessed ' - 'via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting ' - 'of the\n' - 'exception class, the exception instance and a traceback object ' - '(see\n' - 'section The standard type hierarchy) identifying the point in ' - 'the\n' - 'program where the exception occurred. The details about the ' - 'exception\n' - 'accessed via "sys.exc_info()" are restored to their previous ' - 'values\n' - 'when leaving an exception handler:\n' + 'Before an "except" clause’s suite is executed, the exception is ' + 'stored\n' + 'in the "sys" module, where it can be accessed from within the ' + 'body of\n' + 'the "except" clause by calling "sys.exception()". When leaving ' + 'an\n' + 'exception handler, the exception stored in the "sys" module is ' + 'reset\n' + 'to its previous value:\n' '\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' >>> print(sys.exception())\n' + ' None\n' ' >>> try:\n' ' ... raise TypeError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' ' ... try:\n' ' ... raise ValueError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' + ' ... print(repr(sys.exception()))\n' ' ...\n' - " (, TypeError(), )\n' - " (, ValueError(), )\n' - " (, TypeError(), )\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' TypeError()\n' + ' ValueError()\n' + ' TypeError()\n' + ' >>> print(sys.exception())\n' + ' None\n' '\n' '\n' '"except*" clause\n' @@ -4533,7 +4524,7 @@ 'objects and\n' ' implements an "__eq__()" method, it should not ' 'implement\n' - ' "__hash__()", since the implementation of hashable ' + ' "__hash__()", since the implementation of *hashable* ' 'collections\n' ' requires that a key’s hash value is immutable (if the ' 'object’s hash\n' @@ -9453,7 +9444,7 @@ ' hashable collections. If a class defines mutable objects ' 'and\n' ' implements an "__eq__()" method, it should not implement\n' - ' "__hash__()", since the implementation of hashable ' + ' "__hash__()", since the implementation of *hashable* ' 'collections\n' ' requires that a key’s hash value is immutable (if the ' 'object’s hash\n' @@ -12644,37 +12635,31 @@ 'cycle with the stack frame, keeping all locals in that frame alive\n' 'until the next garbage collection occurs.\n' '\n' - 'Before an "except" clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of ' - 'the\n' - 'exception class, the exception instance and a traceback object (see\n' - 'section The standard type hierarchy) identifying the point in the\n' - 'program where the exception occurred. The details about the ' - 'exception\n' - 'accessed via "sys.exc_info()" are restored to their previous values\n' - 'when leaving an exception handler:\n' + 'Before an "except" clause’s suite is executed, the exception is ' + 'stored\n' + 'in the "sys" module, where it can be accessed from within the body ' + 'of\n' + 'the "except" clause by calling "sys.exception()". When leaving an\n' + 'exception handler, the exception stored in the "sys" module is reset\n' + 'to its previous value:\n' '\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' >>> print(sys.exception())\n' + ' None\n' ' >>> try:\n' ' ... raise TypeError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' ' ... try:\n' ' ... raise ValueError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' + ' ... print(repr(sys.exception()))\n' ' ...\n' - " (, TypeError(), )\n' - " (, ValueError(), )\n' - " (, TypeError(), )\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' TypeError()\n' + ' ValueError()\n' + ' TypeError()\n' + ' >>> print(sys.exception())\n' + ' None\n' '\n' '\n' '"except*" clause\n' @@ -14408,7 +14393,7 @@ ' New in version 3.10.\n' '\n' 'Keys views are set-like since their entries are unique and ' - 'hashable.\n' + '*hashable*.\n' 'If all values are hashable, so that "(key, value)" pairs are ' 'unique\n' 'and hashable, then the items view is also set-like. (Values ' diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst new file mode 100644 index 000000000000000..2bcb4c8c854d4e9 --- /dev/null +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -0,0 +1,821 @@ +.. date: 2023-02-17-10-42-48 +.. gh-issue: 99108 +.. nonce: MKA8-f +.. release date: 2023-03-07 +.. section: Security + +Replace builtin hashlib implementations of MD5 and SHA1 with verified ones +from the HACL* project. + +.. + +.. date: 2023-02-08-22-03-04 +.. gh-issue: 101727 +.. nonce: 9P5eZz +.. section: Security + +Updated the OpenSSL version used in Windows and macOS binary release builds +to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +`the OpenSSL 2023-02-07 security advisory +`_. + +.. + +.. date: 2023-02-08-12-57-35 +.. gh-issue: 99108 +.. nonce: 6tnmhA +.. section: Security + +Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 +originally from LibTomCrypt with formally verified, side-channel resistant +code from the `HACL* `_ project. +The builtins remain a fallback only used when OpenSSL does not provide them. + +.. + +.. date: 2023-01-24-16-12-00 +.. gh-issue: 101283 +.. nonce: 9tqu39 +.. section: Security + +:class:`subprocess.Popen` now uses a safer approach to find ``cmd.exe`` when +launching with ``shell=True``. Patch by Eryk Sun, based on a patch by Oleg +Iarygin. + +.. + +.. date: 2023-03-07-16-56-28 +.. gh-issue: 102493 +.. nonce: gTXrcD +.. section: Core and Builtins + +Fix regression in semantics of normalisation in ``PyErr_SetObject``. + +.. + +.. date: 2023-03-06-13-05-33 +.. gh-issue: 102416 +.. nonce: dz6K5f +.. section: Core and Builtins + +Do not memoize incorrectly automatically generated loop rules in the parser. +Patch by Pablo Galindo. + +.. + +.. date: 2023-03-04-20-56-12 +.. gh-issue: 102356 +.. nonce: 07KvUd +.. section: Core and Builtins + +Fix a bug that caused a crash when deallocating deeply nested filter +objects. Patch by Marta Gómez Macías. + +.. + +.. date: 2023-02-28-21-17-03 +.. gh-issue: 102336 +.. nonce: -wL3Tm +.. section: Core and Builtins + +Cleanup Windows 7 specific special handling. Patch by Max Bachmann. + +.. + +.. date: 2023-02-26-23-10-32 +.. gh-issue: 102250 +.. nonce: 7MUKoC +.. section: Core and Builtins + +Fixed a segfault occurring when the interpreter calls a ``__bool__`` method +that raises. + +.. + +.. date: 2023-02-24-17-59-39 +.. gh-issue: 102126 +.. nonce: HTT8Vc +.. section: Core and Builtins + +Fix deadlock at shutdown when clearing thread states if any finalizer tries +to acquire the runtime head lock. Patch by Kumar Aditya. + +.. + +.. date: 2023-02-22-15-15-32 +.. gh-issue: 102027 +.. nonce: Km4G-d +.. section: Core and Builtins + +Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch +by Max Bachmann. + +.. + +.. date: 2023-02-20-15-18-33 +.. gh-issue: 102056 +.. nonce: uHKuwH +.. section: Core and Builtins + +Fix error handling bugs in interpreter's exception printing code, which +could cause a crash on infinite recursion. + +.. + +.. date: 2023-02-17-10-12-13 +.. gh-issue: 100982 +.. nonce: mJGJQw +.. section: Core and Builtins + +Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope +of the original :opcode:`FOR_ITER` instruction, to allow instrumentation. + +.. + +.. date: 2023-02-16-23-19-01 +.. gh-issue: 101967 +.. nonce: Kqr1dz +.. section: Core and Builtins + +Fix possible segfault in ``positional_only_passed_as_keyword`` function, +when new list created. + +.. + +.. date: 2023-02-16-16-57-23 +.. gh-issue: 101952 +.. nonce: Zo1dlq +.. section: Core and Builtins + +Fix possible segfault in ``BUILD_SET`` opcode, when new set created. + +.. + +.. date: 2023-02-13-22-21-58 +.. gh-issue: 74895 +.. nonce: esMNtq +.. section: Core and Builtins + +:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for +:class:`int` **port** values outside of the C long range. Out of range +values are left up to the underlying string based C library API to report. A +:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at +all as not all platform C libraries generate an error. + +.. + +.. date: 2023-02-13-18-21-14 +.. gh-issue: 101799 +.. nonce: wpHbCn +.. section: Core and Builtins + +Add :opcode:`CALL_INTRINSIC_2` and use it instead of +:opcode:`PREP_RERAISE_STAR`. + +.. + +.. date: 2023-02-12-22-40-22 +.. gh-issue: 101857 +.. nonce: _bribG +.. section: Core and Builtins + +Fix xattr support detection on Linux systems by widening the check to linux, +not just glibc. This fixes support for musl. + +.. + +.. date: 2023-02-11-23-14-06 +.. gh-issue: 84783 +.. nonce: _P5sMa +.. section: Core and Builtins + +Make the slice object hashable. + +.. + +.. date: 2023-02-10-15-54-57 +.. gh-issue: 87849 +.. nonce: IUVvPz +.. section: Core and Builtins + +Change the ``SEND`` instruction to leave the receiver on the stack. This +allows the specialized form of ``SEND`` to skip the chain of C calls and +jump directly to the ``RESUME`` in the generator or coroutine. + +.. + +.. date: 2023-02-10-07-21-47 +.. gh-issue: 101765 +.. nonce: MO5LlC +.. section: Core and Builtins + +Fix SystemError / segmentation fault in iter ``__reduce__`` when internal +access of ``builtins.__dict__`` keys mutates the iter object. + +.. + +.. date: 2023-02-10-01-15-57 +.. gh-issue: 101430 +.. nonce: T3Gegb +.. section: Core and Builtins + +Update :mod:`tracemalloc` to handle presize of object properly. Patch by +Dong-hee Na. + +.. + +.. date: 2023-02-08-17-13-31 +.. gh-issue: 101696 +.. nonce: seJhTt +.. section: Core and Builtins + +Invalidate type version tag in ``_PyStaticType_Dealloc`` for static types, +avoiding bug where a false cache hit could crash the interpreter. Patch by +Kumar Aditya. + +.. + +.. date: 2023-02-07-14-56-43 +.. gh-issue: 101632 +.. nonce: Fd1yxk +.. section: Core and Builtins + +Adds a new :opcode:`RETURN_CONST` instruction. + +.. + +.. date: 2023-01-04-12-49-33 +.. gh-issue: 100719 +.. nonce: uRPccL +.. section: Core and Builtins + +Remove gi_code field from generator (and coroutine and async generator) +objects as it is redundant. The frame already includes a reference to the +code object. + +.. + +.. date: 2022-11-02-20-23-47 +.. gh-issue: 98627 +.. nonce: VJkdRM +.. section: Core and Builtins + +When an interpreter is configured to check (and only then), importing an +extension module will now fail when the extension does not support multiple +interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does +not apply to the main interpreter, nor to subinterpreters created with +``Py_NewInterpreter()``. + +.. + +.. date: 2023-03-04-14-46-47 +.. gh-issue: 102302 +.. nonce: -b_s6Z +.. section: Library + +Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it +takes to hash an instance by around 40%. + +.. + +.. date: 2023-02-28-09-52-25 +.. gh-issue: 101979 +.. nonce: or3hXV +.. section: Library + +Fix a bug where parentheses in the ``metavar`` argument to +:meth:`argparse.ArgumentParser.add_argument` were dropped. Patch by Yeojin +Kim. + +.. + +.. date: 2023-02-26-12-37-17 +.. gh-issue: 91038 +.. nonce: S4rFH_ +.. section: Library + +:meth:`platform.platform` now has boolean default arguments. + +.. + +.. date: 2023-02-23-20-39-52 +.. gh-issue: 81652 +.. nonce: Vxz0Mr +.. section: Library + +Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` +OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. + +.. + +.. date: 2023-02-23-15-06-01 +.. gh-issue: 102179 +.. nonce: P6KQ4c +.. section: Library + +Fix :func:`os.dup2` error message for negative fds. + +.. + +.. date: 2023-02-21-10-05-33 +.. gh-issue: 101961 +.. nonce: 7e56jh +.. section: Library + +For the binary mode, :func:`fileinput.hookcompressed` doesn't set the +``encoding`` value even if the value is ``None``. Patch by Gihwan Kim. + +.. + +.. date: 2023-02-21-07-15-41 +.. gh-issue: 101936 +.. nonce: QVOxHH +.. section: Library + +The default value of ``fp`` becomes :class:`io.BytesIO` if +:exc:`~urllib.error.HTTPError` is initialized without a designated ``fp`` +parameter. Patch by Long Vo. + +.. + +.. date: 2023-02-17-20-24-15 +.. gh-issue: 101566 +.. nonce: FjgWBt +.. section: Library + +In zipfile, sync Path with `zipp 3.14 +`_, including +fix for extractall on the underlying zipfile after being wrapped in +``Path``. + +.. + +.. date: 2023-02-17-19-00-58 +.. gh-issue: 97930 +.. nonce: C_nQjb +.. section: Library + +Apply changes from `importlib_resources 5.12 +`_, +including fix for ``MultiplexedPath`` to support directories in multiple +namespaces (python/importlib_resources#265). + +.. + +.. date: 2023-02-17-18-44-27 +.. gh-issue: 101997 +.. nonce: A6_blD +.. section: Library + +Upgrade pip wheel bundled with ensurepip (pip 23.0.1) + +.. + +.. date: 2023-02-15-01-54-06 +.. gh-issue: 99108 +.. nonce: rjTSic +.. section: Library + +The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when +OpenSSL does not provide them, now live in a single internal ``_sha2`` +module instead of separate ``_sha256`` and ``_sha512`` modules. + +.. + +.. date: 2023-02-14-09-08-48 +.. gh-issue: 101892 +.. nonce: FMos8l +.. section: Library + +Callable iterators no longer raise :class:`SystemError` when the callable +object exhausts the iterator but forgets to either return a sentinel value +or raise :class:`StopIteration`. + +.. + +.. date: 2023-02-13-12-55-48 +.. gh-issue: 87634 +.. nonce: q-SBhJ +.. section: Library + +Remove locking behavior from :func:`functools.cached_property`. + +.. + +.. date: 2023-02-11-13-23-29 +.. gh-issue: 97786 +.. nonce: QjvQ1B +.. section: Library + +Fix potential undefined behaviour in corner cases of floating-point-to-time +conversions. + +.. + +.. date: 2023-02-10-16-02-29 +.. gh-issue: 101517 +.. nonce: r7S2u8 +.. section: Library + +Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` +with a ``lineno=None``, which causes it to fail with an unhandled exception. + +.. + +.. date: 2023-02-10-11-59-13 +.. gh-issue: 101773 +.. nonce: J_kI7y +.. section: Library + +Optimize :class:`fractions.Fraction` for small components. The private +argument ``_normalize`` of the :class:`fractions.Fraction` constructor has +been removed. + +.. + +.. date: 2023-02-08-18-20-58 +.. gh-issue: 101693 +.. nonce: 4_LPXj +.. section: Library + +In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted +when :ref:`named placeholders ` are used together with +parameters supplied as a :term:`sequence` instead of as a :class:`dict`. +Starting from Python 3.14, using named placeholders with parameters supplied +as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. Patch by Erlend +E. Aasland. + +.. + +.. date: 2023-02-07-22-21-46 +.. gh-issue: 101446 +.. nonce: -c0FdK +.. section: Library + +Change repr of :class:`collections.OrderedDict` to use regular dictionary +formating instead of pairs of keys and values. + +.. + +.. date: 2023-02-07-22-20-32 +.. gh-issue: 101362 +.. nonce: Jlk6mt +.. section: Library + +Speed up :class:`pathlib.PurePath` construction by handling arguments more +uniformly. When a :class:`pathlib.Path` argument is supplied, we use its +string representation rather than joining its parts with +:func:`os.path.join`. + +.. + +.. date: 2023-02-07-21-16-41 +.. gh-issue: 101362 +.. nonce: KMQllM +.. section: Library + +Speed up :class:`pathlib.PurePath` construction by calling +:func:`os.path.join` only when two or more arguments are given. + +.. + +.. date: 2023-02-07-20-46-08 +.. gh-issue: 101362 +.. nonce: 2ckZ6R +.. section: Library + +Speed up :class:`pathlib.Path` construction by running the path flavour +compatibility check only when pathlib is imported. + +.. + +.. date: 2023-02-05-21-40-15 +.. gh-issue: 85984 +.. nonce: Kfzbb2 +.. section: Library + +Refactored the implementation of :func:`pty.fork` to use +:func:`os.login_tty`. + +A :exc:`DeprecationWarning` is now raised by ``pty.master_open()`` and +``pty.slave_open()``. They were undocumented and deprecated long long ago in +the docstring in favor of :func:`pty.openpty`. + +.. + +.. date: 2023-02-04-16-35-46 +.. gh-issue: 101561 +.. nonce: Xo6pIZ +.. section: Library + +Add a new decorator :func:`typing.override`. See :pep:`698` for details. +Patch by Steven Troxler. + +.. + +.. date: 2023-02-01-10-42-16 +.. gh-issue: 63301 +.. nonce: XNxSFh +.. section: Library + +Set exit code when :mod:`tabnanny` CLI exits on error. + +.. + +.. date: 2023-01-27-02-53-50 +.. gh-issue: 101360 +.. nonce: bPB7SL +.. section: Library + +Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and +pattern anchors are now matched with :mod:`fnmatch`, just like other path +parts. This allows patterns such as ``"*:/Users/*"`` to be matched. + +.. + +.. date: 2023-01-25-00-14-52 +.. gh-issue: 101277 +.. nonce: FceHX7 +.. section: Library + +Remove global state from :mod:`itertools` module (:pep:`687`). Patches by +Erlend E. Aasland. + +.. + +.. date: 2023-01-06-21-14-41 +.. gh-issue: 100809 +.. nonce: I697UT +.. section: Library + +Fix handling of drive-relative paths (like 'C:' and 'C:foo') in +:meth:`pathlib.Path.absolute`. This method now uses the OS API to retrieve +the correct current working directory for the drive. + +.. + +.. date: 2023-01-02-22-41-44 +.. gh-issue: 99138 +.. nonce: 17hp9U +.. section: Library + +Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. + +.. + +.. date: 2022-10-22-09-26-43 +.. gh-issue: 96764 +.. nonce: Dh9Y5L +.. section: Library + +:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying +implementation. Patch by Kumar Aditya. + +.. + +.. date: 2022-09-05-12-17-34 +.. gh-issue: 88233 +.. nonce: gff9qJ +.. section: Library + +Correctly preserve "extra" fields in ``zipfile`` regardless of their +ordering relative to a zip64 "extra." + +.. + +.. bpo: 23224 +.. date: 2018-06-20-09-12-21 +.. nonce: zxCQ13 +.. section: Library + +Fix segfaults when creating :class:`lzma.LZMADecompressor` and +:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix +leakage of locks and internal buffers when calling the ``__init__()`` +methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, +:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects +multiple times. + +.. + +.. date: 2023-02-19-10-33-01 +.. gh-issue: 85417 +.. nonce: kYO8u3 +.. section: Documentation + +Update :mod:`cmath` documentation to clarify behaviour on branch cuts. + +.. + +.. date: 2023-02-07-21-43-24 +.. gh-issue: 97725 +.. nonce: cuY7Cd +.. section: Documentation + +Fix :meth:`asyncio.Task.print_stack` description for ``file=None``. Patch by +Oleg Iarygin. + +.. + +.. date: 2023-02-18-10-51-02 +.. gh-issue: 102019 +.. nonce: 0797SJ +.. section: Tests + +Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch +by Jacob Bower. + +.. + +.. date: 2023-02-11-22-36-10 +.. gh-issue: 85984 +.. nonce: EVXjT9 +.. section: Tests + +Utilize new "winsize" functions from termios in pty tests. + +.. + +.. date: 2023-02-11-20-28-08 +.. gh-issue: 89792 +.. nonce: S-Y5BZ +.. section: Tests + +``test_tools`` now copies up to 10x less source data to a temporary +directory during the ``freeze`` test by ignoring git metadata and other +artifacts. It also limits its python build parallelism based on +os.cpu_count instead of hard coding it as 8 cores. + +.. + +.. date: 2023-01-12-00-49-16 +.. gh-issue: 99942 +.. nonce: DUR8b4 +.. section: Build + +On Android, in a static build, python-config in embed mode no longer +incorrectly reports a library to link to. + +.. + +.. date: 2022-12-20-01-06-17 +.. gh-issue: 99942 +.. nonce: lbmzYj +.. section: Build + +On Android, python.pc now correctly reports the library to link to, the same +as python-config.sh. + +.. + +.. date: 2022-12-18-08-33-28 +.. gh-issue: 100221 +.. nonce: K94Ct3 +.. section: Build + +Fix creating install directories in ``make sharedinstall`` if they exist +outside ``DESTDIR`` already. + +.. + +.. date: 2022-09-14-10-38-15 +.. gh-issue: 96821 +.. nonce: Zk2a9c +.. section: Build + +Explicitly mark C extension modules that need defined signed integer +overflow, and add a configure option :option:`--with-strict-overflow`. Patch +by Matthias Görgens and Shantanu Jain. + +.. + +.. date: 2023-03-01-01-36-39 +.. gh-issue: 102344 +.. nonce: Dgfux4 +.. section: Windows + +Implement ``winreg.QueryValue`` using ``QueryValueEx`` and +``winreg.SetValue`` using ``SetValueEx``. Patch by Max Bachmann. + +.. + +.. date: 2023-02-15-11-08-10 +.. gh-issue: 101881 +.. nonce: fScr3m +.. section: Windows + +Handle read and write operations on non-blocking pipes properly on Windows. + +.. + +.. date: 2023-02-13-18-05-49 +.. gh-issue: 101881 +.. nonce: _TnHzN +.. section: Windows + +Add support for the os.get_blocking() and os.set_blocking() functions on +Windows. + +.. + +.. date: 2023-02-13-16-32-50 +.. gh-issue: 101849 +.. nonce: 7lm_53 +.. section: Windows + +Ensures installer will correctly upgrade existing ``py.exe`` launcher +installs. + +.. + +.. date: 2023-02-10-14-26-05 +.. gh-issue: 101763 +.. nonce: RPaj7r +.. section: Windows + +Updates copy of libffi bundled with Windows installs to 3.4.4. + +.. + +.. date: 2023-02-09-22-09-27 +.. gh-issue: 101759 +.. nonce: zFlqSH +.. section: Windows + +Update Windows installer to SQLite 3.40.1. + +.. + +.. date: 2023-02-07-18-22-54 +.. gh-issue: 101614 +.. nonce: NjVP0n +.. section: Windows + +Correctly handle extensions built against debug binaries that reference +``python3_d.dll``. + +.. + +.. date: 2023-01-25-11-33-54 +.. gh-issue: 101196 +.. nonce: wAX_2g +.. section: Windows + +The functions ``os.path.isdir``, ``os.path.isfile``, ``os.path.islink`` and +``os.path.exists`` are now 13% to 28% faster on Windows, by making fewer +Win32 API calls. + +.. + +.. date: 2023-02-09-22-07-17 +.. gh-issue: 101759 +.. nonce: B0JP2H +.. section: macOS + +Update macOS installer to SQLite 3.40.1. + +.. + +.. date: 2023-02-14-15-53-01 +.. gh-issue: 101907 +.. nonce: HgF1N2 +.. section: C API + +Removes use of non-standard C++ extension in public header files. + +.. + +.. date: 2023-02-09-10-38-20 +.. gh-issue: 99293 +.. nonce: mFqfpp +.. section: C API + +Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, +should not be used, and will be removed. + +.. + +.. date: 2023-02-06-16-14-30 +.. gh-issue: 101578 +.. nonce: PW5fA9 +.. section: C API + +Add :c:func:`PyErr_GetRaisedException` and +:c:func:`PyErr_SetRaisedException` for saving and restoring the current +exception. These functions return and accept a single exception object, +rather than the triple arguments of the now-deprecated :c:func:`PyErr_Fetch` +and :c:func:`PyErr_Restore`. This is less error prone and a bit more +efficient. + +Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` as +convenience functions for retrieving and modifying the +:attr:`~BaseException.args` passed to the exception's constructor. + +.. + +.. date: 2022-04-21-17-25-22 +.. gh-issue: 91744 +.. nonce: FgvaMi +.. section: C API + +Introduced the *Unstable C API tier*, marking APi that is allowed to change +in minor releases without a deprecation period. See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst deleted file mode 100644 index 865cfde8b063598..000000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst +++ /dev/null @@ -1,3 +0,0 @@ -Explicitly mark C extension modules that need defined signed integer overflow, -and add a configure option :option:`--with-strict-overflow`. -Patch by Matthias Görgens and Shantanu Jain. diff --git a/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst b/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst deleted file mode 100644 index 27c948330cfc17b..000000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix creating install directories in ``make sharedinstall`` if they exist -outside ``DESTDIR`` already. diff --git a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst deleted file mode 100644 index 63a640a9cdf7338..000000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Android, python.pc now correctly reports the library to link to, the same -as python-config.sh. diff --git a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst deleted file mode 100644 index 5b692c3cc458c56..000000000000000 --- a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Android, in a static build, python-config in embed mode no longer -incorrectly reports a library to link to. diff --git a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst deleted file mode 100644 index 20db25ddd0c1f6d..000000000000000 --- a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Introduced the *Unstable C API tier*, marking APi that is allowed to change -in minor releases without a deprecation period. -See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst b/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst deleted file mode 100644 index 27294a9e5179c48..000000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst +++ /dev/null @@ -1,10 +0,0 @@ -Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` -for saving and restoring the current exception. -These functions return and accept a single exception object, -rather than the triple arguments of the now-deprecated -:c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`. -This is less error prone and a bit more efficient. - -Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` -as convenience functions for retrieving and modifying -the :attr:`~BaseException.args` passed to the exception's constructor. diff --git a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst b/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst deleted file mode 100644 index 8c0f05543747dc0..000000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, -should not be used, and will be removed. diff --git a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst b/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst deleted file mode 100644 index cfc0d72cdbca003..000000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst +++ /dev/null @@ -1 +0,0 @@ -Removes use of non-standard C++ extension in public header files. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst deleted file mode 100644 index 3d2d6f6eb0c41f5..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst +++ /dev/null @@ -1,5 +0,0 @@ -When an interpreter is configured to check (and only then), importing an -extension module will now fail when the extension does not support multiple -interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does -not apply to the main interpreter, nor to subinterpreters created with -``Py_NewInterpreter()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst deleted file mode 100644 index 2addef27b8ea4dc..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove gi_code field from generator (and coroutine and async generator) -objects as it is redundant. The frame already includes a reference to the -code object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst deleted file mode 100644 index 136909ca6999030..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst +++ /dev/null @@ -1 +0,0 @@ -Adds a new :opcode:`RETURN_CONST` instruction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst deleted file mode 100644 index ff2bbb4b5642522..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst +++ /dev/null @@ -1 +0,0 @@ -Invalidate type version tag in ``_PyStaticType_Dealloc`` for static types, avoiding bug where a false cache hit could crash the interpreter. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst deleted file mode 100644 index e617d85242144ea..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`tracemalloc` to handle presize of object properly. Patch by -Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst deleted file mode 100644 index cc99779a944ec6d..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst deleted file mode 100644 index da5f3ff79fd575c..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the ``SEND`` instruction to leave the receiver on the stack. This -allows the specialized form of ``SEND`` to skip the chain of C calls and jump -directly to the ``RESUME`` in the generator or coroutine. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst deleted file mode 100644 index e1c851a0825a7f0..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst +++ /dev/null @@ -1 +0,0 @@ -Make the slice object hashable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst deleted file mode 100644 index 832cc300fa9433a..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst +++ /dev/null @@ -1 +0,0 @@ -Fix xattr support detection on Linux systems by widening the check to linux, not just glibc. This fixes support for musl. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst deleted file mode 100644 index 3233a573be7acdf..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :opcode:`CALL_INTRINSIC_2` and use it instead of -:opcode:`PREP_RERAISE_STAR`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst deleted file mode 100644 index adbbb601634a602..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for -:class:`int` **port** values outside of the C long range. Out of range values -are left up to the underlying string based C library API to report. A -:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at all -as not all platform C libraries generate an error. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst deleted file mode 100644 index 3902c988c8bf9ff..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible segfault in ``BUILD_SET`` opcode, when new set created. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst deleted file mode 100644 index 6e681f910f53597..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible segfault in ``positional_only_passed_as_keyword`` function, when new list created. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst deleted file mode 100644 index 53bbc860c53f374..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope of the -original :opcode:`FOR_ITER` instruction, to allow instrumentation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst deleted file mode 100644 index 78cd525b365fe5e..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix error handling bugs in interpreter's exception printing code, which could cause a crash on infinite recursion. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst deleted file mode 100644 index 514a8ef26594dce..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch by -Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst deleted file mode 100644 index 68c43688c3df035..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst +++ /dev/null @@ -1 +0,0 @@ -Fix deadlock at shutdown when clearing thread states if any finalizer tries to acquire the runtime head lock. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst deleted file mode 100644 index 17ab0cd43679912..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a segfault occurring when the interpreter calls a ``__bool__`` method that raises. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst deleted file mode 100644 index 0c3e4bd4b860948..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst +++ /dev/null @@ -1 +0,0 @@ -Cleanup Windows 7 specific special handling. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst deleted file mode 100644 index c03fd5266bc3017..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug that caused a crash when deallocating deeply nested filter -objects. Patch by Marta Gómez Macías. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst deleted file mode 100644 index 9ffc67cfb7ed56a..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst +++ /dev/null @@ -1 +0,0 @@ -Do not memoize incorrectly automatically generated loop rules in the parser. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst deleted file mode 100644 index 4c4e88ca4e7c3cd..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix regression in semantics of normalisation in ``PyErr_SetObject``. diff --git a/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst b/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst deleted file mode 100644 index fd9ea049c239687..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.Task.print_stack` description for ``file=None``. -Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst deleted file mode 100644 index a5532df14795d22..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst +++ /dev/null @@ -1 +0,0 @@ -Update :mod:`cmath` documentation to clarify behaviour on branch cuts. diff --git a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst b/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst deleted file mode 100644 index 8909753c7f9ee62..000000000000000 --- a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix segfaults when creating :class:`lzma.LZMADecompressor` and -:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix -leakage of locks and internal buffers when calling the ``__init__()`` -methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, -:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects -multiple times. diff --git a/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst b/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst deleted file mode 100644 index 806f7011edc3986..000000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly preserve "extra" fields in ``zipfile`` regardless of their -ordering relative to a zip64 "extra." diff --git a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst deleted file mode 100644 index a0174291cbc3110..000000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying implementation. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst deleted file mode 100644 index 3dd4646f40e1e54..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst +++ /dev/null @@ -1 +0,0 @@ -Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst deleted file mode 100644 index 54082de88ccf4ab..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix handling of drive-relative paths (like 'C:' and 'C:foo') in -:meth:`pathlib.Path.absolute`. This method now uses the OS API -to retrieve the correct current working directory for the drive. diff --git a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst b/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst deleted file mode 100644 index e09c0e09fb388fe..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove global state from :mod:`itertools` module (:pep:`687`). Patches by -Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst deleted file mode 100644 index 4cfb136c5db8530..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and -pattern anchors are now matched with :mod:`fnmatch`, just like other path -parts. This allows patterns such as ``"*:/Users/*"`` to be matched. diff --git a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst deleted file mode 100644 index e00e71fb8554f35..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst +++ /dev/null @@ -1 +0,0 @@ -Set exit code when :mod:`tabnanny` CLI exits on error. diff --git a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst b/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst deleted file mode 100644 index 2f6a4153062e5a1..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new decorator :func:`typing.override`. See :pep:`698` for details. Patch by Steven Troxler. diff --git a/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst b/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst deleted file mode 100644 index c91829f2c739afb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst +++ /dev/null @@ -1,4 +0,0 @@ -Refactored the implementation of :func:`pty.fork` to use :func:`os.login_tty`. - -A :exc:`DeprecationWarning` is now raised by ``pty.master_open()`` and ``pty.slave_open()``. They were -undocumented and deprecated long long ago in the docstring in favor of :func:`pty.openpty`. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst deleted file mode 100644 index 8421466cdbb3c9f..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.Path` construction by running the path flavour -compatibility check only when pathlib is imported. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst deleted file mode 100644 index af4ee9ad904868e..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.PurePath` construction by calling -:func:`os.path.join` only when two or more arguments are given. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst deleted file mode 100644 index c05f92ae699de96..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up :class:`pathlib.PurePath` construction by handling arguments more -uniformly. When a :class:`pathlib.Path` argument is supplied, -we use its string representation rather than joining its parts -with :func:`os.path.join`. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst deleted file mode 100644 index ddf897b71bb1d14..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. diff --git a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst deleted file mode 100644 index e436054b15b6576..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst +++ /dev/null @@ -1,6 +0,0 @@ -In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted -when :ref:`named placeholders ` are used together with -parameters supplied as a :term:`sequence` instead of as a :class:`dict`. -Starting from Python 3.14, using named placeholders with parameters supplied -as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst b/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst deleted file mode 100644 index b577d93d28c2ae9..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :class:`fractions.Fraction` for small components. The private argument -``_normalize`` of the :class:`fractions.Fraction` constructor has been removed. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst b/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst deleted file mode 100644 index a5f6bdfa5ac2f0e..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` with a ``lineno=None``, which causes it to fail with an unhandled exception. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst deleted file mode 100644 index df194b67590d67d..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential undefined behaviour in corner cases of floating-point-to-time -conversions. diff --git a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst b/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst deleted file mode 100644 index a17927500bd9a54..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst +++ /dev/null @@ -1 +0,0 @@ -Remove locking behavior from :func:`functools.cached_property`. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst deleted file mode 100644 index d586779b3a8a365..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst +++ /dev/null @@ -1,3 +0,0 @@ -Callable iterators no longer raise :class:`SystemError` when the -callable object exhausts the iterator but forgets to either return a -sentinel value or raise :class:`StopIteration`. diff --git a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst deleted file mode 100644 index 1612c89c0ea6bee..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst +++ /dev/null @@ -1,3 +0,0 @@ -The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when -OpenSSL does not provide them, now live in a single internal ``_sha2`` module -instead of separate ``_sha256`` and ``_sha512`` modules. diff --git a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst b/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst deleted file mode 100644 index f9dfd46d1ed4304..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade pip wheel bundled with ensurepip (pip 23.0.1) diff --git a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst deleted file mode 100644 index 967e13f752bcd12..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst +++ /dev/null @@ -1,4 +0,0 @@ -Apply changes from `importlib_resources 5.12 -`_, -including fix for ``MultiplexedPath`` to support directories in multiple -namespaces (python/importlib_resources#265). diff --git a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst b/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst deleted file mode 100644 index 5fc1a0288a82dcb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst +++ /dev/null @@ -1,4 +0,0 @@ -In zipfile, sync Path with `zipp 3.14 -`_, including -fix for extractall on the underlying zipfile after being wrapped in -``Path``. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst deleted file mode 100644 index 55841da44b11461..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst +++ /dev/null @@ -1,2 +0,0 @@ -The default value of ``fp`` becomes :class:`io.BytesIO` if :exc:`~urllib.error.HTTPError` -is initialized without a designated ``fp`` parameter. Patch by Long Vo. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst deleted file mode 100644 index a3d4119e7cbdceb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst +++ /dev/null @@ -1,2 +0,0 @@ -For the binary mode, :func:`fileinput.hookcompressed` doesn't set the ``encoding`` value -even if the value is ``None``. Patch by Gihwan Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst b/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst deleted file mode 100644 index f77493e267ac7eb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`os.dup2` error message for negative fds. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst b/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst deleted file mode 100644 index 48acce1d863ea60..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` -OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst b/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst deleted file mode 100644 index 2667ff120fd402e..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`platform.platform` now has boolean default arguments. diff --git a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst deleted file mode 100644 index 1efe72439b3a4ad..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug where parentheses in the ``metavar`` argument to :meth:`argparse.ArgumentParser.add_argument` were -dropped. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst b/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst deleted file mode 100644 index aaf4e62069ca248..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst +++ /dev/null @@ -1 +0,0 @@ -Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it takes to hash an instance by around 40%. diff --git a/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst b/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst deleted file mode 100644 index 0efdfa102341856..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`subprocess.Popen` now uses a safer approach to find -``cmd.exe`` when launching with ``shell=True``. Patch by Eryk Sun, -based on a patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst deleted file mode 100644 index 6a7a309dad5d8f9..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 -originally from LibTomCrypt with formally verified, side-channel resistant -code from the `HACL* `_ project. -The builtins remain a fallback only used when OpenSSL does not provide them. diff --git a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst b/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst deleted file mode 100644 index 43acc82063fd7ae..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Updated the OpenSSL version used in Windows and macOS binary release builds -to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per -`the OpenSSL 2023-02-07 security advisory -`_. diff --git a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst deleted file mode 100644 index 723d8a43a09f9ef..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Replace builtin hashlib implementations of MD5 and SHA1 with verified ones -from the HACL* project. diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst deleted file mode 100644 index 9de278919ef2f8a..000000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst +++ /dev/null @@ -1,4 +0,0 @@ -``test_tools`` now copies up to 10x less source data to a temporary directory -during the ``freeze`` test by ignoring git metadata and other artifacts. It -also limits its python build parallelism based on os.cpu_count instead of hard -coding it as 8 cores. diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst b/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst deleted file mode 100644 index 402f99ea6c6ebfb..000000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst +++ /dev/null @@ -1 +0,0 @@ -Utilize new "winsize" functions from termios in pty tests. diff --git a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst b/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst deleted file mode 100644 index 63e36046d26dfee..000000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch -by Jacob Bower. diff --git a/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst b/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst deleted file mode 100644 index c61e9b90fb53732..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst +++ /dev/null @@ -1,3 +0,0 @@ -The functions ``os.path.isdir``, ``os.path.isfile``, ``os.path.islink`` and -``os.path.exists`` are now 13% to 28% faster on Windows, by making fewer Win32 -API calls. diff --git a/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst b/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst deleted file mode 100644 index 8ed0995d78925ba..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst +++ /dev/null @@ -1 +0,0 @@ -Correctly handle extensions built against debug binaries that reference ``python3_d.dll``. diff --git a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst b/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst deleted file mode 100644 index 62bcac34397d2e0..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to SQLite 3.40.1. diff --git a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst b/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst deleted file mode 100644 index e7e5a73afeb5320..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst +++ /dev/null @@ -1 +0,0 @@ -Updates copy of libffi bundled with Windows installs to 3.4.4. diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst deleted file mode 100644 index 861d4de9f9a650a..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst +++ /dev/null @@ -1 +0,0 @@ -Ensures installer will correctly upgrade existing ``py.exe`` launcher installs. diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst deleted file mode 100644 index ba58dd4f5cb450e..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for the os.get_blocking() and os.set_blocking() functions on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst deleted file mode 100644 index 099b2c1c07a6659..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst +++ /dev/null @@ -1 +0,0 @@ -Handle read and write operations on non-blocking pipes properly on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst b/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst deleted file mode 100644 index 4804212be8182c8..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement ``winreg.QueryValue`` using ``QueryValueEx`` and -``winreg.SetValue`` using ``SetValueEx``. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst b/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst deleted file mode 100644 index fc53d08bffc4fd4..000000000000000 --- a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.40.1. diff --git a/README.rst b/README.rst index b1756e20c141ab0..6923b692f6c9717 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 5 +This is Python version 3.12.0 alpha 6 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From 1e703a473343ed198c9a06a876b25d7d69d4bbd0 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 7 Mar 2023 17:10:58 -0700 Subject: [PATCH 03/35] gh-102381: don't call watcher callback with dead object (#102382) Co-authored-by: T. Wouters --- Doc/c-api/code.rst | 17 ++++++++-- Doc/c-api/dict.rst | 21 +++++++++--- Doc/c-api/function.rst | 17 ++++++++-- Include/cpython/code.h | 13 +++++--- Include/cpython/dictobject.h | 21 +++++++----- Include/cpython/funcobject.h | 16 ++++----- Include/internal/pycore_dict.h | 1 + Lib/test/test_capi/test_watchers.py | 52 +++++++++++++++++++++++++++-- Modules/_testcapi/watchers.c | 13 +++++++- Objects/codeobject.c | 38 ++++++++++++++++++++- Objects/dictobject.c | 34 +++++++++++++++++-- Objects/funcobject.c | 38 ++++++++++++++++++++- 12 files changed, 243 insertions(+), 38 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 062ef3a1fea93cd..a99de9904c07408 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -172,6 +172,11 @@ bound into a function. before the destruction of *co* takes place, so the prior state of *co* can be inspected. + If *event* is ``PY_CODE_EVENT_DESTROY``, taking a reference in the callback + to the about-to-be-destroyed code object will resurrect it and prevent it + from being freed at this time. When the resurrected object is destroyed + later, any watcher callbacks active at that time will be called again. + Users of this API should not rely on internal runtime implementation details. Such details may include, but are not limited to, the exact order and timing of creation and destruction of code objects. While @@ -179,9 +184,15 @@ bound into a function. (including whether a callback is invoked or not), it does not change the semantics of the Python code being executed. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 34106ee6b1f30df..b9f84cea7856440 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -298,13 +298,26 @@ Dictionary Objects dictionary. The callback may inspect but must not modify *dict*; doing so could have - unpredictable effects, including infinite recursion. + unpredictable effects, including infinite recursion. Do not trigger Python + code execution in the callback, as it could modify the dict as a side effect. + + If *event* is ``PyDict_EVENT_DEALLOCATED``, taking a new reference in the + callback to the about-to-be-destroyed dictionary will resurrect it and + prevent it from being freed at this time. When the resurrected object is + destroyed later, any watcher callbacks active at that time will be called + again. Callbacks occur before the notified modification to *dict* takes place, so the prior state of *dict* can be inspected. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index bc7569d0add97db..947ed70404081b3 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -173,8 +173,19 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + callback to the about-to-be-destroyed function will resurrect it, preventing + it from being freed at this time. When the resurrected object is destroyed + later, any watcher callbacks active at that time will be called again. + + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 0e4bd8a58c165b3..abcf1250603dfed 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -224,9 +224,14 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *); -typedef enum PyCodeEvent { - PY_CODE_EVENT_CREATE, - PY_CODE_EVENT_DESTROY +#define PY_FOREACH_CODE_EVENT(V) \ + V(CREATE) \ + V(DESTROY) + +typedef enum { + #define PY_DEF_EVENT(op) PY_CODE_EVENT_##op, + PY_FOREACH_CODE_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyCodeEvent; @@ -236,7 +241,7 @@ typedef enum PyCodeEvent { * The callback is invoked with a borrowed reference to co, after it is * created and before it is destroyed. * - * If the callback returns with an exception set, it must return -1. Otherwise + * If the callback sets an exception, it must return -1. Otherwise * it should return 0. */ typedef int (*PyCode_WatchCallback)( diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 5001f35654475e6..ddada922020aa4c 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -16,11 +16,11 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ -#ifdef Py_BUILD_CORE +#ifdef Py_BUILD_CORE uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif +#endif PyDictKeysObject *ma_keys; @@ -90,13 +90,18 @@ PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); /* Dictionary watchers */ +#define PY_FOREACH_DICT_EVENT(V) \ + V(ADDED) \ + V(MODIFIED) \ + V(DELETED) \ + V(CLONED) \ + V(CLEARED) \ + V(DEALLOCATED) + typedef enum { - PyDict_EVENT_ADDED, - PyDict_EVENT_MODIFIED, - PyDict_EVENT_DELETED, - PyDict_EVENT_CLONED, - PyDict_EVENT_CLEARED, - PyDict_EVENT_DEALLOCATED, + #define PY_DEF_EVENT(EVENT) PyDict_EVENT_##EVENT, + PY_FOREACH_DICT_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyDict_WatchEvent; // Callback to be invoked when a watched dict is cleared, dealloced, or modified. diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 5979febc2e3421e..c716330cc3fbab9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -131,17 +131,17 @@ PyAPI_DATA(PyTypeObject) PyStaticMethod_Type; PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *); PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); -#define FOREACH_FUNC_EVENT(V) \ - V(CREATE) \ - V(DESTROY) \ - V(MODIFY_CODE) \ - V(MODIFY_DEFAULTS) \ +#define PY_FOREACH_FUNC_EVENT(V) \ + V(CREATE) \ + V(DESTROY) \ + V(MODIFY_CODE) \ + V(MODIFY_DEFAULTS) \ V(MODIFY_KWDEFAULTS) typedef enum { - #define DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, - FOREACH_FUNC_EVENT(DEF_EVENT) - #undef DEF_EVENT + #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, + PY_FOREACH_FUNC_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyFunction_WatchEvent; /* diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index c74a3437713039f..1af5e59a677a9a1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -164,6 +164,7 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, PyObject *key, PyObject *value) { + assert(Py_REFCNT((PyObject*)mp) > 0); int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 1922614ef60558a..93f6ef752d0663c 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -109,10 +109,21 @@ def test_error(self): self.watch(wid, d) with catch_unraisable_exception() as cm: d["foo"] = "bar" - self.assertIs(cm.unraisable.object, d) + self.assertIn( + "PyDict_EVENT_ADDED watcher callback for 0); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->_initialized); uint8_t bits = interp->active_code_watchers; @@ -25,7 +40,21 @@ notify_code_watchers(PyCodeEvent event, PyCodeObject *co) // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, co) < 0) { - PyErr_WriteUnraisable((PyObject *) co); + // Don't risk resurrecting the object if an unraisablehook keeps + // a reference; pass a string as context. + PyObject *context = NULL; + PyObject *repr = code_repr(co); + if (repr) { + context = PyUnicode_FromFormat( + "%s watcher callback for %U", + code_event_name(event), repr); + Py_DECREF(repr); + } + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } i++; @@ -1667,7 +1696,14 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, static void code_dealloc(PyCodeObject *co) { + assert(Py_REFCNT(co) == 0); + Py_SET_REFCNT(co, 1); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); + if (Py_REFCNT(co) > 1) { + Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + return; + } + Py_SET_REFCNT(co, 0); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index fc658ca2f4b7f86..e3795e75e3da5ee 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2308,7 +2308,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + assert(Py_REFCNT(mp) == 0); + Py_SET_REFCNT(mp, 1); _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + if (Py_REFCNT(mp) > 1) { + Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); + return; + } + Py_SET_REFCNT(mp, 0); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -5732,6 +5739,18 @@ PyDict_ClearWatcher(int watcher_id) return 0; } +static const char * +dict_event_name(PyDict_WatchEvent event) { + switch (event) { + #define CASE(op) \ + case PyDict_EVENT_##op: \ + return "PyDict_EVENT_" #op; + PY_FOREACH_DICT_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + void _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, @@ -5744,9 +5763,18 @@ _PyDict_SendEvent(int watcher_bits, if (watcher_bits & 1) { PyDict_WatchCallback cb = interp->dict_state.watchers[i]; if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { - // some dict modification paths (e.g. PyDict_Clear) can't raise, so we - // can't propagate exceptions from dict watchers. - PyErr_WriteUnraisable((PyObject *)mp); + // We don't want to resurrect the dict by potentially having an + // unraisablehook keep a reference to it, so we don't pass the + // dict as context, just an informative string message. Dict + // repr can call arbitrary code, so we invent a simpler version. + PyObject *context = PyUnicode_FromFormat( + "%s watcher callback for ", + dict_event_name(event), mp); + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } watcher_bits >>= 1; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 91a6b3dd40a2324..99048ea41c6c800 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -8,6 +8,20 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "structmember.h" // PyMemberDef +static PyObject* func_repr(PyFunctionObject *op); + +static const char * +func_event_name(PyFunction_WatchEvent event) { + switch (event) { + #define CASE(op) \ + case PyFunction_EVENT_##op: \ + return "PyFunction_EVENT_" #op; + PY_FOREACH_FUNC_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + static void notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -21,7 +35,21 @@ notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, func, new_value) < 0) { - PyErr_WriteUnraisable((PyObject *) func); + // Don't risk resurrecting the func if an unraisablehook keeps a + // reference; pass a string as context. + PyObject *context = NULL; + PyObject *repr = func_repr(func); + if (repr != NULL) { + context = PyUnicode_FromFormat( + "%s watcher callback for %U", + func_event_name(event), repr); + Py_DECREF(repr); + } + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } i++; @@ -33,6 +61,7 @@ static inline void handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { + assert(Py_REFCNT(func) > 0); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->_initialized); if (interp->active_func_watchers) { @@ -766,7 +795,14 @@ func_clear(PyFunctionObject *op) static void func_dealloc(PyFunctionObject *op) { + assert(Py_REFCNT(op) == 0); + Py_SET_REFCNT(op, 1); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); + if (Py_REFCNT(op) > 1) { + Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + return; + } + Py_SET_REFCNT(op, 0); _PyObject_GC_UNTRACK(op); if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); From e499680100a9d4fa4b673cbcfebc32212b4f848a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 8 Mar 2023 09:09:50 +0530 Subject: [PATCH 04/35] fix typo in async generator code field name `ag_code` (#102448) --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 4ab6581e12ab3aa..be08a59ece6b7e7 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1549,7 +1549,7 @@ ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) static PyObject * ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) { - return _gen_getcode(gen, "ag__code"); + return _gen_getcode(gen, "ag_code"); } static PyGetSetDef async_gen_getsetlist[] = { From 02b9a921cb3e6308ad300ee9c398b24998220931 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 8 Mar 2023 05:04:38 +0100 Subject: [PATCH 05/35] Post 3.12.0a6 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index fd16421eace94af..049cdfa30897cac 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.12.0a6" +#define PY_VERSION "3.12.0a6+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 061325e0d2bbec6ff89d03f527c91dc7bfa14003 Mon Sep 17 00:00:00 2001 From: Marcin Wieczorek Date: Wed, 8 Mar 2023 08:25:28 +0100 Subject: [PATCH 06/35] Fix style in argparse.rst (#101733) --- Doc/library/argparse.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 34b4c61649b99f4..ee68ac58d3de753 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -34,9 +34,9 @@ around an instance of :class:`argparse.ArgumentParser`. It is a container for argument specifications and has options that apply to the parser as whole:: parser = argparse.ArgumentParser( - prog = 'ProgramName', - description = 'What the program does', - epilog = 'Text at the bottom of help') + prog='ProgramName', + description='What the program does', + epilog='Text at the bottom of help') The :meth:`ArgumentParser.add_argument` method attaches individual argument specifications to the parser. It supports positional arguments, options that From 1a84cc007e207f2dd61f86a7fc3d86632fdce72f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:29:39 +0530 Subject: [PATCH 07/35] GH-102397: Fix segfault from race condition in signal handling (#102399) Co-authored-by: Gregory P. Smith --- Lib/test/test_signal.py | 15 +++++++++++++++ ...2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst | 2 ++ Modules/signalmodule.c | 4 ++++ 3 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 2562a57ea421ffd..25afd6aabe07519 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1406,6 +1406,21 @@ def handler(a, b): signal.raise_signal(signal.SIGINT) self.assertTrue(is_ok) + def test__thread_interrupt_main(self): + # See https://github.com/python/cpython/issues/102397 + code = """if 1: + import _thread + class Foo(): + def __del__(self): + _thread.interrupt_main() + + x = Foo() + """ + + rc, out, err = assert_python_ok('-c', code) + self.assertIn(b'OSError: Signal 2 ignored due to race condition', err) + + class PidfdSignalTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst new file mode 100644 index 000000000000000..db0b3f32c2ec0b6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst @@ -0,0 +1,2 @@ +Fix segfault from race condition in signal handling during garbage collection. +Patch by Kumar Aditya. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index cd26eca351c0ed5..0e472e1ee4f9ddc 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -148,6 +148,10 @@ get_signal_state(PyObject *module) static inline int compare_handler(PyObject *func, PyObject *dfl_ign_handler) { + // See https://github.com/python/cpython/pull/102399 + if (func == NULL || dfl_ign_handler == NULL) { + return 0; + } assert(PyLong_CheckExact(dfl_ign_handler)); if (!PyLong_CheckExact(func)) { return 0; From 1f557f94c2ee98c2a43bd090a7bf3f39a22ed874 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 8 Mar 2023 11:19:05 +0300 Subject: [PATCH 08/35] gh-101100: Fix sphinx warnings in `zipapp` and `zipfile` modules (#102526) --- Doc/library/zipapp.rst | 2 +- Doc/library/zipfile.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index fb40a2b3e964e4d..981020b13cd988f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -215,7 +215,7 @@ using the :func:`create_archive` function:: >>> import zipapp >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3') -To update the file in place, do the replacement in memory using a :class:`BytesIO` +To update the file in place, do the replacement in memory using a :class:`~io.BytesIO` object, and then overwrite the source afterwards. Note that there is a risk when overwriting a file in place that an error will result in the loss of the original file. This code does not protect against such errors, but diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 0195abc3a992c1b..e2a085d6e98e670 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -288,7 +288,7 @@ ZipFile Objects (``ZipExtFile``) is read-only and provides the following methods: :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`, - :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`. + :meth:`~io.IOBase.tell`, :meth:`~container.__iter__`, :meth:`~iterator.__next__`. These objects can operate independently of the ZipFile. With ``mode='w'``, a writable file handle is returned, which supports the From 7d801f245e2021d19daff105ce722f22aa844391 Mon Sep 17 00:00:00 2001 From: sblondon Date: Wed, 8 Mar 2023 11:24:39 +0100 Subject: [PATCH 09/35] Remove or update bitbucket links (GH-101963) Since Mercurial removal from bitbucket.org, some links are broken. They are replaced by github.com or webarchive.org links if available. Otherwise, they are removed. Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/distributing/index.rst | 3 +-- Doc/installing/index.rst | 3 +-- Doc/library/venv.rst | 2 +- Lib/test/test_pathlib.py | 2 +- PC/launcher.c | 3 +-- PC/launcher2.c | 3 +-- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 2ae2726d4e4b92a..21389adedf9c154 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -39,8 +39,7 @@ Key terms developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub `__ and - `Bitbucket `__. + and issue trackers on `GitHub `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index e158bf1c4c0c7f5..5aec5178d48f3df 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -52,8 +52,7 @@ Key terms developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, - and issue trackers on both `GitHub `__ and - `Bitbucket `__. + and issue trackers on `GitHub `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2a41096de006b80..8eb0b35eaa12df6 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -478,7 +478,7 @@ subclass which installs setuptools and pip into a created virtual environment:: :param context: The information for the virtual environment creation request being processed. """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' + url = "https://bootstrap.pypa.io/ez_setup.py" self.install_script(context, 'setuptools', url) # clear up the setuptools archive which gets downloaded pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index df9c1f6ba65deb5..f8e2f44d27fc1ef 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1938,7 +1938,7 @@ def test_resolve_common(self): @os_helper.skip_unless_symlink def test_resolve_dot(self): - # See https://bitbucket.org/pitrou/pathlib/issue/9/pathresolve-fails-on-complex-symlinks + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ p = self.cls(BASE) self.dirlink('.', join('0')) self.dirlink(os.path.join('0', '0'), join('1')) diff --git a/PC/launcher.c b/PC/launcher.c index da566a180168c5d..0776e57249c427b 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -770,8 +770,7 @@ run_child(wchar_t * cmdline) window, or fetching a message). As this launcher doesn't do this directly, that cursor remains even after the child process does these things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 and - https://bitbucket.org/vinay.sajip/pylauncher/issue/20/busy-cursor-for-a-long-time-when-running + See http://bugs.python.org/issue17290 */ MSG msg; diff --git a/PC/launcher2.c b/PC/launcher2.c index beeb2ae46b83f04..932665387f19666 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -2473,8 +2473,7 @@ launchEnvironment(const SearchInfo *search, const EnvironmentInfo *launch, wchar window, or fetching a message). As this launcher doesn't do this directly, that cursor remains even after the child process does these things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 and - https://bitbucket.org/vinay.sajip/pylauncher/issue/20/busy-cursor-for-a-long-time-when-running + See http://bugs.python.org/issue17290 */ MSG msg; From 401d7a7f009ca2e282b1a0d1b880dc602afd39dc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 8 Mar 2023 11:45:38 +0000 Subject: [PATCH 10/35] gh-102515: Remove unused imports in the `Lib/` directory (#102516) --- Lib/_pylong.py | 1 - Lib/concurrent/futures/process.py | 2 ++ Lib/dataclasses.py | 1 - Lib/importlib/_abc.py | 1 - Lib/sysconfig.py | 2 +- Lib/test/_test_venv_multiprocessing.py | 1 - Lib/test/fork_wait.py | 2 +- Lib/test/test__xxinterpchannels.py | 1 - Lib/test/test__xxsubinterpreters.py | 2 -- Lib/test/test_asyncgen.py | 1 - Lib/test/test_asyncio/test_ssl.py | 1 - Lib/test/test_asyncio/test_subprocess.py | 1 - Lib/test/test_ctypes/test_callbacks.py | 1 - Lib/test/test_dataclasses.py | 1 - Lib/test/test_enum.py | 2 -- Lib/test/test_grammar.py | 2 -- Lib/test/test_hashlib.py | 1 - Lib/test/test_httplib.py | 2 -- Lib/test/test_imaplib.py | 2 -- Lib/test/test_isinstance.py | 1 - Lib/test/test_minidom.py | 1 - Lib/test/test_peg_generator/test_c_parser.py | 1 - Lib/test/test_shlex.py | 1 - Lib/test/test_sys_setprofile.py | 1 - Lib/test/test_sys_settrace.py | 1 - Lib/test/test_tkinter/test_widgets.py | 3 +-- Lib/test/test_tools/test_sundry.py | 1 - Lib/test/test_traceback.py | 1 - Lib/test/test_ttk/test_widgets.py | 4 +--- Lib/test/test_unicodedata.py | 3 +-- Lib/test/test_unittest/test_loader.py | 1 - Lib/test/test_wmi.py | 1 - Lib/test/test_zipfile64.py | 2 +- 33 files changed, 8 insertions(+), 41 deletions(-) diff --git a/Lib/_pylong.py b/Lib/_pylong.py index d14c1d938363279..936346e187ff699 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -12,7 +12,6 @@ tricky or non-obvious code is not worth it. For people looking for maximum performance, they should use something like gmpy2.""" -import sys import re import decimal diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index bee162430a6f8e1..3a8637b6fa1b47f 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -49,6 +49,8 @@ from concurrent.futures import _base import queue import multiprocessing as mp +# This import is required to load the multiprocessing.connection submodule +# so that it can be accessed later as `mp.connection` import multiprocessing.connection from multiprocessing.queues import Queue import threading diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 5c0257eba186d15..8bc8594d674bc0d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,7 +4,6 @@ import types import inspect import keyword -import builtins import functools import itertools import abc diff --git a/Lib/importlib/_abc.py b/Lib/importlib/_abc.py index 083205638965215..693b466112638f5 100644 --- a/Lib/importlib/_abc.py +++ b/Lib/importlib/_abc.py @@ -1,7 +1,6 @@ """Subset of importlib.abc used to reduce importlib.util imports.""" from . import _bootstrap import abc -import warnings class Loader(metaclass=abc.ABCMeta): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index c61100a6da85033..122d441bd19f5e8 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -3,7 +3,7 @@ import os import sys import threading -from os.path import pardir, realpath +from os.path import realpath __all__ = [ 'get_config_h_filename', diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py index 044a0c6cd3f5ca0..ad985dd8d56bb4b 100644 --- a/Lib/test/_test_venv_multiprocessing.py +++ b/Lib/test/_test_venv_multiprocessing.py @@ -1,7 +1,6 @@ import multiprocessing import random import sys -import time def fill_queue(queue, code): queue.put(code) diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index c26c7aaaeb43061..8c32895f5e09e5d 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -9,7 +9,7 @@ active threads survive in the child after a fork(); this is an error. """ -import os, sys, time, unittest +import os, time, unittest import threading from test import support from test.support import threading_helper diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 03bb5c80b8dac92..69bda89a1688f57 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -1,6 +1,5 @@ from collections import namedtuple import contextlib -import os import sys from textwrap import dedent import threading diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 687fcf3b770522a..965967e3f2734b4 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1,4 +1,3 @@ -from collections import namedtuple import contextlib import itertools import os @@ -6,7 +5,6 @@ import sys from textwrap import dedent import threading -import time import unittest import _testcapi diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 0421efdbf9dac9a..09e4010b0e53d6f 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -2,7 +2,6 @@ import types import unittest import contextlib -import warnings from test.support.import_helper import import_module from test.support import gc_collect, requires_working_socket diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index aaf3c37101f52a6..e9cc735613fb8e1 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1,5 +1,4 @@ import asyncio -import asyncio.sslproto import contextlib import gc import logging diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index f1ad10a9903fe86..eba6e2d1f28f3ec 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,5 +1,4 @@ import os -import shutil import signal import sys import unittest diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index e8fa3e6f7aca514..b185e388ab1527d 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import sys import functools import unittest from test import support diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 81a36aa241acf76..589f229f4623594 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2261,7 +2261,6 @@ def test_base_has_init(self): class B: def __init__(self): self.z = 100 - pass # Make sure that declaring this class doesn't raise an error. # The issue is that we can't override __init__ in our class, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0a2e0c14d268af1..2b14590b2c21af5 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -20,7 +20,6 @@ from test import support from test.support import ALWAYS_EQ from test.support import threading_helper -from textwrap import dedent from datetime import timedelta python_version = sys.version_info[:2] @@ -1187,7 +1186,6 @@ class MyEnum(HexInt, enum.Enum): # class SillyInt(HexInt): __qualname__ = 'SillyInt' - pass class MyOtherEnum(SillyInt, enum.Enum): __qualname__ = 'MyOtherEnum' D = 4 diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 5b946020994e31e..ced9000f75f2e57 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -3,7 +3,6 @@ from test.support import check_syntax_error from test.support import import_helper -from test.support.warnings_helper import check_syntax_warning import inspect import unittest import sys @@ -15,7 +14,6 @@ # with import machinery import test.ann_module as ann_module import typing -from collections import ChainMap from test import ann_module2 import test diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5ead8857943592d..08cb5eb0c2bbab8 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -22,7 +22,6 @@ from test.support import os_helper from test.support import requires_resource from test.support import threading_helper -from test.support import warnings_helper from http.client import HTTPException diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 620a5b19109a8c1..9ff6afcbadec54c 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -8,7 +8,6 @@ import re import socket import threading -import warnings import unittest from unittest import mock @@ -17,7 +16,6 @@ from test import support from test.support import os_helper from test.support import socket_helper -from test.support import warnings_helper support.requires_working_socket(module=True) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 7626d9572e1e969..60f5b671b1da485 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -13,7 +13,6 @@ from test.support import verbose, run_with_tz, run_with_locale, cpython_only from test.support import hashlib_helper from test.support import threading_helper -from test.support import warnings_helper import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -751,7 +750,6 @@ class NonUTF8Server(SimpleIMAPHandler): typ, data = client.login('user', 'pass') self.assertEqual(typ, 'OK') client.enable('UTF8=ACCEPT') - pass @threading_helper.reap_threads def test_enable_UTF8_True_append(self): diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 2fcf6ebbee7e346..bf9332e40aeaf2e 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -3,7 +3,6 @@ # testing of error conditions uncovered when using extension types. import unittest -import sys import typing from test import support diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 2ca3908bd1caac1..699265ccadc7f9c 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -6,7 +6,6 @@ from test import support import unittest -import pyexpat import xml.dom.minidom from xml.dom.minidom import parse, Attr, Node, Document, parseString diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1b3fcbb92f82923..d34ffef0dbc5ecc 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -24,7 +24,6 @@ generate_parser_c_extension, generate_c_parser_source, ) - from pegen.ast_dump import ast_dump TEST_TEMPLATE = """ diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 92598dbbd5f293c..797c91ee7effdfd 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -3,7 +3,6 @@ import shlex import string import unittest -from unittest import mock # The original test data set was from shellwords, by Hartmut Goebel. diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index acae433cd0a549d..49e076c77d167ae 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -2,7 +2,6 @@ import pprint import sys import unittest -from test import support class TestGetProfile(unittest.TestCase): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb0..4907c930e143d50 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2,7 +2,6 @@ from test import support import unittest -from unittest.mock import MagicMock import sys import difflib import gc diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 6fde93cbecc73ff..64c9472706549bb 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -9,8 +9,7 @@ AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, - setUpModule) + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 81f06763980a325..6a3dc12781b2b6b 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -6,7 +6,6 @@ """ import os -import sys import unittest from test.support import import_helper diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 95b1bae4f60850b..92c5a000585855e 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -6,7 +6,6 @@ import sys import types import inspect -import importlib import builtins import unittest import re diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 6f47ccb8e8b3ded..79d65b496abdc64 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -8,8 +8,7 @@ from test.test_tkinter.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, - setUpModule) + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') @@ -50,7 +49,6 @@ def test_configure_style(self): widget2 = self.create(class_='Foo') self.assertEqual(widget2['class'], 'Foo') # XXX - pass class WidgetTest(AbstractTkTest, unittest.TestCase): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 74503c89e559a04..3dc0790ca15b416 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,8 +12,7 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation, - ResourceDenied) + cpython_only, check_disallow_instantiation) class UnicodeMethodsTest(unittest.TestCase): diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index bbdfb247ebaada4..a203145a791b1a0 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,7 +1,6 @@ import functools import sys import types -import warnings import unittest diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 3f5795952905244..3445702846d8a04 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -1,7 +1,6 @@ # Test the internal _wmi module on Windows # This is used by the platform module, and potentially others -import sys import unittest from test.support import import_helper, requires_resource diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index be654a8478b04b7..2e1affe02528581 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,7 +11,7 @@ 'test requires loads of disk-space bytes and a long time to run' ) -import zipfile, os, unittest +import zipfile, unittest import time import sys From b097925858c6975c73e989226cf278cc382c0416 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:58:14 +0000 Subject: [PATCH 11/35] gh-102507 Remove invisible pagebreak characters (#102531) Co-authored-by: AlexWaygood --- Lib/email/__init__.py | 1 - Lib/email/base64mime.py | 4 ---- Lib/email/charset.py | 5 ----- Lib/email/encoders.py | 4 ---- Lib/email/feedparser.py | 2 -- Lib/email/generator.py | 5 +---- Lib/email/header.py | 5 ----- Lib/email/iterators.py | 3 --- Lib/email/mime/base.py | 1 - Lib/email/mime/message.py | 1 - Lib/email/mime/multipart.py | 1 - Lib/email/mime/nonmultipart.py | 1 - Lib/email/mime/text.py | 1 - Lib/email/parser.py | 3 +-- Modules/_io/bufferedio.c | 3 --- Tools/i18n/pygettext.py | 11 ++++------- 16 files changed, 6 insertions(+), 45 deletions(-) diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index fae872439edc66d..9fa477830041859 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -25,7 +25,6 @@ ] - # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index a7cc37365c6f9a1..4cdf22666e30161 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -45,7 +45,6 @@ MISC_LEN = 7 - # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" @@ -57,7 +56,6 @@ def header_length(bytearray): return n - def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. @@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'): return '=?%s?b?%s?=' % (charset, encoded) - def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. @@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL): return EMPTYSTRING.join(encvec) - def decode(string): """Decode a raw base64 string, returning a bytes object. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 791b6584b247572..9af269442fb8afb 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -18,7 +18,6 @@ from email.encoders import encode_7or8bit - # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 @@ -32,7 +31,6 @@ EMPTYSTRING = '' - # Defaults CHARSETS = { # input header enc body enc output conv @@ -104,7 +102,6 @@ } - # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. @@ -153,7 +150,6 @@ def add_codec(charset, codecname): CODEC_MAP[charset] = codecname - # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): @@ -163,7 +159,6 @@ def _encode(string, codec): return string.encode(codec) - class Charset: """Map character sets to their email properties. diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0a66acb6240bd78..17bd1ab7b19f325 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -16,7 +16,6 @@ from quopri import encodestring as _encodestring - def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do @@ -34,7 +33,6 @@ def encode_base64(msg): msg['Content-Transfer-Encoding'] = 'base64' - def encode_quopri(msg): """Encode the message's payload in quoted-printable. @@ -46,7 +44,6 @@ def encode_quopri(msg): msg['Content-Transfer-Encoding'] = 'quoted-printable' - def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) @@ -64,6 +61,5 @@ def encode_7or8bit(msg): msg['Content-Transfer-Encoding'] = '7bit' - def encode_noop(msg): """Do nothing.""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 97d3f5144d606ff..6bc4e0c4e598957 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -41,7 +41,6 @@ NeedMoreData = object() - class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. @@ -132,7 +131,6 @@ def __next__(self): return line - class FeedParser: """A feed-style parser of email.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 885e6ba98540a7d..7ccbe10eb768567 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -22,7 +22,6 @@ fcre = re.compile(r'^From ', re.MULTILINE) - class Generator: """Generates output from a Message object tree. @@ -392,7 +391,7 @@ def _make_boundary(cls, text=None): def _compile_re(cls, s, flags): return re.compile(s, flags) - + class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. @@ -443,7 +442,6 @@ def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) - _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): @@ -503,7 +501,6 @@ def _dispatch(self, msg): }, file=self) - # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width diff --git a/Lib/email/header.py b/Lib/email/header.py index 4ab0032bc661234..984851a7d9a679b 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -52,12 +52,10 @@ _embedded_header = re.compile(r'\n[^ \t]+:') - # Helpers _max_append = email.quoprimime._max_append - def decode_header(header): """Decode a message header value without converting charset. @@ -152,7 +150,6 @@ def decode_header(header): return collapsed - def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() @@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h - class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, @@ -409,7 +405,6 @@ def _normalize(self): self._chunks = chunks - class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index b5502ee975266ba..3410935e38f4760 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -15,7 +15,6 @@ from io import StringIO - # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. @@ -29,7 +28,6 @@ def walk(self): yield from subpart.walk() - # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. @@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None): yield subpart - def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index 1a3f9b51f6c0456..f601f621cec3933 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -11,7 +11,6 @@ from email import message - class MIMEBase(message.Message): """Base class for MIME specializations.""" diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 07e4f2d11961516..61836b5a7861fca 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEMessage(MIMENonMultipart): """Class representing message/* MIME documents.""" diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 2d3f288810dd919..94d81c771a474eb 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -9,7 +9,6 @@ from email.mime.base import MIMEBase - class MIMEMultipart(MIMEBase): """Base class for MIME multipart/* type messages.""" diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index e1f51968b59eb15..a41386eb148c0c8 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -10,7 +10,6 @@ from email.mime.base import MIMEBase - class MIMENonMultipart(MIMEBase): """Base class for MIME non-multipart type messages.""" diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 35b442383002b22..dfe53c426b2ac40 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEText(MIMENonMultipart): """Class for generating text/* type MIME documents.""" diff --git a/Lib/email/parser.py b/Lib/email/parser.py index e94d455baa52622..06d99b17f2f9c49 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -64,7 +64,6 @@ def parsestr(self, text, headersonly=False): return self.parse(StringIO(text), headersonly=headersonly) - class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) @@ -72,7 +71,7 @@ def parse(self, fp, headersonly=True): def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) - + class BytesParser: def __init__(self, *args, **kw): diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 960026707fc5ed7..2c71768be978702 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1742,7 +1742,6 @@ _bufferedreader_peek_unlocked(buffered *self) self->pos = 0; return PyBytes_FromStringAndSize(self->buffer, r); } - /* @@ -2052,7 +2051,6 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) LEAVE_BUFFERED(self) return res; } - /* @@ -2266,7 +2264,6 @@ bufferedrwpair_closed_get(rwpair *self, void *context) } return PyObject_GetAttr((PyObject *) self->writer, &_Py_ID(closed)); } - /* diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 7ada79105db1cad..3a0b27ba420e7a7 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -174,7 +174,6 @@ EMPTYSTRING = '' - # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's # there. pot_header = _('''\ @@ -196,7 +195,7 @@ ''') - + def usage(code, msg=''): print(__doc__ % globals(), file=sys.stderr) if msg: @@ -204,7 +203,6 @@ def usage(code, msg=''): sys.exit(code) - def make_escapes(pass_nonascii): global escapes, escape if pass_nonascii: @@ -258,7 +256,7 @@ def normalize(s, encoding): s = '""\n"' + lineterm.join(lines) + '"' return s - + def containsAny(str, set): """Check whether 'str' contains ANY of the chars in 'set'""" return 1 in [c in str for c in set] @@ -307,7 +305,7 @@ def getFilesForName(name): return [] - + class TokenEater: def __init__(self, options): self.__options = options @@ -515,7 +513,6 @@ def write(self, fp): print('msgstr ""\n', file=fp) - def main(): global default_keywords try: @@ -675,7 +672,7 @@ class Options: if closep: fp.close() - + if __name__ == '__main__': main() # some more test strings From 11a2c6ce516b24b2435cb627742a6c4df92d411c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:03:18 +0000 Subject: [PATCH 12/35] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (in Objects/) (#102218) --- Objects/dictobject.c | 2 +- Objects/exceptions.c | 39 +++++++++---------------- Objects/frameobject.c | 5 ++-- Objects/genobject.c | 64 +++++++++++------------------------------ Objects/object.c | 25 +++++++--------- Objects/odictobject.c | 5 ++-- Objects/typeobject.c | 10 +++---- Objects/weakrefobject.c | 7 ++--- 8 files changed, 51 insertions(+), 106 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e3795e75e3da5ee..75c92172a91778c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -119,7 +119,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "stringlib/eq.h" // unicode_eq() diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a473cbdfeda7fc1..c6fb6a3f19b2d06 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3781,16 +3781,13 @@ PyObject * _PyErr_TrySetFromCause(const char *format, ...) { PyObject* msg_prefix; - PyObject *exc, *val, *tb; - PyTypeObject *caught_type; PyObject *instance_args; Py_ssize_t num_args, caught_type_size, base_exc_size; - PyObject *new_exc, *new_val, *new_tb; va_list vargs; int same_basic_size; - PyErr_Fetch(&exc, &val, &tb); - caught_type = (PyTypeObject *)exc; + PyObject *exc = PyErr_GetRaisedException(); + PyTypeObject *caught_type = Py_TYPE(exc); /* Ensure type info indicates no extra state is stored at the C level * and that the type can be reinstantiated using PyErr_Format */ @@ -3810,31 +3807,30 @@ _PyErr_TrySetFromCause(const char *format, ...) * more state than just the exception type. Accordingly, we just * leave it alone. */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } /* Check the args are empty or contain a single string */ - PyErr_NormalizeException(&exc, &val, &tb); - instance_args = ((PyBaseExceptionObject *)val)->args; + instance_args = ((PyBaseExceptionObject *)exc)->args; num_args = PyTuple_GET_SIZE(instance_args); if (num_args > 1 || (num_args == 1 && !PyUnicode_CheckExact(PyTuple_GET_ITEM(instance_args, 0)))) { /* More than 1 arg, or the one arg we do have isn't a string */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } /* Ensure the instance dict is also empty */ - if (!_PyObject_IsInstanceDictEmpty(val)) { + if (!_PyObject_IsInstanceDictEmpty(exc)) { /* While we could potentially copy a non-empty instance dictionary * to the replacement exception, for now we take the more * conservative path of leaving exceptions with attributes set * alone. */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } @@ -3847,28 +3843,19 @@ _PyErr_TrySetFromCause(const char *format, ...) * types as well, but that's quite a bit trickier due to the extra * state potentially stored on OSError instances. */ - /* Ensure the traceback is set correctly on the existing exception */ - if (tb != NULL) { - PyException_SetTraceback(val, tb); - Py_DECREF(tb); - } - va_start(vargs, format); msg_prefix = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg_prefix == NULL) { Py_DECREF(exc); - Py_DECREF(val); return NULL; } - PyErr_Format(exc, "%U (%s: %S)", - msg_prefix, Py_TYPE(val)->tp_name, val); - Py_DECREF(exc); + PyErr_Format((PyObject*)Py_TYPE(exc), "%U (%s: %S)", + msg_prefix, Py_TYPE(exc)->tp_name, exc); Py_DECREF(msg_prefix); - PyErr_Fetch(&new_exc, &new_val, &new_tb); - PyErr_NormalizeException(&new_exc, &new_val, &new_tb); - PyException_SetCause(new_val, val); - PyErr_Restore(new_exc, new_val, new_tb); - return new_val; + PyObject *new_exc = PyErr_GetRaisedException(); + PyException_SetCause(new_exc, exc); + PyErr_SetRaisedException(new_exc); + return new_exc; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 34143c9a40b293e..133c991bf701c4e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1308,7 +1308,6 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) /* Merge locals into fast locals */ PyObject *locals; PyObject **fast; - PyObject *error_type, *error_value, *error_traceback; PyCodeObject *co; locals = frame->f_locals; if (locals == NULL) { @@ -1317,7 +1316,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) fast = _PyFrame_GetLocalsArray(frame); co = frame->f_code; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -1374,7 +1373,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) } Py_XDECREF(value); } - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } void diff --git a/Objects/genobject.c b/Objects/genobject.c index be08a59ece6b7e7..61463774310f884 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -69,8 +69,6 @@ void _PyGen_Finalize(PyObject *self) { PyGenObject *gen = (PyGenObject *)self; - PyObject *res = NULL; - PyObject *error_type, *error_value, *error_traceback; if (gen->gi_frame_state >= FRAME_COMPLETED) { /* Generator isn't paused, so no need to close */ @@ -82,23 +80,22 @@ _PyGen_Finalize(PyObject *self) PyObject *finalizer = agen->ag_origin_or_finalizer; if (finalizer && !agen->ag_closed) { /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); - - res = PyObject_CallOneArg(finalizer, self); + PyObject *exc = PyErr_GetRaisedException(); + PyObject *res = PyObject_CallOneArg(finalizer, self); if (res == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(res); } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); return; } } /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* If `gen` is a coroutine, and if it was never awaited on, issue a RuntimeWarning. */ @@ -109,20 +106,19 @@ _PyGen_Finalize(PyObject *self) _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } else { - res = gen_close(gen, NULL); - } - - if (res == NULL) { - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(self); + PyObject *res = gen_close(gen, NULL); + if (res == NULL) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } + else { + Py_DECREF(res); } - } - else { - Py_DECREF(res); } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static void @@ -648,39 +644,11 @@ _PyGen_SetStopIterationValue(PyObject *value) int _PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; PyObject *value = NULL; - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Fetch(&et, &ev, &tb); - if (ev) { - /* exception will usually be normalised already */ - if (PyObject_TypeCheck(ev, (PyTypeObject *) et)) { - value = Py_NewRef(((PyStopIterationObject *)ev)->value); - Py_DECREF(ev); - } else if (et == PyExc_StopIteration && !PyTuple_Check(ev)) { - /* Avoid normalisation and take ev as value. - * - * Normalization is required if the value is a tuple, in - * that case the value of StopIteration would be set to - * the first element of the tuple. - * - * (See _PyErr_CreateException code for details.) - */ - value = ev; - } else { - /* normalisation required */ - PyErr_NormalizeException(&et, &ev, &tb); - if (!PyObject_TypeCheck(ev, (PyTypeObject *)PyExc_StopIteration)) { - PyErr_Restore(et, ev, tb); - return -1; - } - value = Py_NewRef(((PyStopIterationObject *)ev)->value); - Py_DECREF(ev); - } - } - Py_XDECREF(et); - Py_XDECREF(tb); + PyObject *exc = PyErr_GetRaisedException(); + value = Py_NewRef(((PyStopIterationObject *)exc)->value); + Py_DECREF(exc); } else if (PyErr_Occurred()) { return -1; } diff --git a/Objects/object.c b/Objects/object.c index 446c7b1f5f0302e..5db2b6af21ef135 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -370,13 +370,12 @@ _PyObject_Dump(PyObject* op) fflush(stderr); PyGILState_STATE gil = PyGILState_Ensure(); - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); (void)PyObject_Print(op, stderr, 0); fflush(stderr); - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); PyGILState_Release(gil); fprintf(stderr, "\n"); @@ -860,25 +859,22 @@ set_attribute_error_context(PyObject* v, PyObject* name) return 0; } // Intercept AttributeError exceptions and augment them to offer suggestions later. - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - PyErr_NormalizeException(&type, &value, &traceback); - // Check if the normalized exception is indeed an AttributeError - if (!PyErr_GivenExceptionMatches(value, PyExc_AttributeError)) { + PyObject *exc = PyErr_GetRaisedException(); + if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) { goto restore; } - PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) value; + PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) exc; // Check if this exception was already augmented if (the_exc->name || the_exc->obj) { goto restore; } // Augment the exception with the name and object - if (PyObject_SetAttr(value, &_Py_ID(name), name) || - PyObject_SetAttr(value, &_Py_ID(obj), v)) { + if (PyObject_SetAttr(exc, &_Py_ID(name), name) || + PyObject_SetAttr(exc, &_Py_ID(obj), v)) { return 1; } restore: - PyErr_Restore(type, value, traceback); + PyErr_SetRaisedException(exc); return 0; } @@ -2190,9 +2186,8 @@ Py_ReprLeave(PyObject *obj) PyObject *dict; PyObject *list; Py_ssize_t i; - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); dict = PyThreadState_GetDict(); if (dict == NULL) @@ -2213,7 +2208,7 @@ Py_ReprLeave(PyObject *obj) finally: /* ignore exceptions because there is no way to report them. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } /* Trashcan support. */ diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 215a8af54fb266b..39b0f684510578b 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1556,10 +1556,9 @@ _PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, res = _odict_add_new_node((PyODictObject *)od, key, hash); if (res < 0) { /* Revert setting the value on the dict */ - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); (void) _PyDict_DelItem_KnownHash(od, key, hash); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); } } return res; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 981930f58417d8a..f486b83fd69e648 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4397,10 +4397,9 @@ static void type_dealloc_common(PyTypeObject *type) { if (type->tp_bases != NULL) { - PyObject *tp, *val, *tb; - PyErr_Fetch(&tp, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); remove_all_subclasses(type, type->tp_bases); - PyErr_Restore(tp, val, tb); + PyErr_SetRaisedException(exc); } } @@ -8445,10 +8444,9 @@ slot_tp_finalize(PyObject *self) { int unbound; PyObject *del, *res; - PyObject *error_type, *error_value, *error_traceback; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* Execute __del__ method, if any. */ del = lookup_maybe_method(self, &_Py_ID(__del__), &unbound); @@ -8462,7 +8460,7 @@ slot_tp_finalize(PyObject *self) } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static PyObject * diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index bd7720e27533078..5a3e49a6fe45e33 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -959,9 +959,8 @@ PyObject_ClearWeakRefs(PyObject *object) if (*list != NULL) { PyWeakReference *current = *list; Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *err_type, *err_value, *err_tb; + PyObject *exc = PyErr_GetRaisedException(); - PyErr_Fetch(&err_type, &err_value, &err_tb); if (count == 1) { PyObject *callback = current->wr_callback; @@ -980,7 +979,7 @@ PyObject_ClearWeakRefs(PyObject *object) tuple = PyTuple_New(count * 2); if (tuple == NULL) { - _PyErr_ChainExceptions(err_type, err_value, err_tb); + _PyErr_ChainExceptions1(exc); return; } @@ -1010,7 +1009,7 @@ PyObject_ClearWeakRefs(PyObject *object) Py_DECREF(tuple); } assert(!PyErr_Occurred()); - PyErr_Restore(err_type, err_value, err_tb); + PyErr_SetRaisedException(exc); } } From cbb0aa71d040022db61390380b8aebc7c04f3275 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 12:03:50 -0700 Subject: [PATCH 13/35] gh-102304: Consolidate Direct Usage of _Py_RefTotal (gh-102514) This simplifies further changes to _Py_RefTotal (e.g. make it atomic or move it to PyInterpreterState). https://github.com/python/cpython/issues/102304 --- Include/cpython/object.h | 1 + Include/internal/pycore_object.h | 23 ++++++++-- Include/object.h | 25 ++++++++-- Modules/_testcapimodule.c | 7 +-- Objects/bytesobject.c | 9 ++-- Objects/dictobject.c | 10 ++-- Objects/object.c | 79 +++++++++++++++++++++++++++----- Objects/structseq.c | 2 +- Objects/tupleobject.c | 9 ++-- Objects/unicodeobject.c | 7 +-- 10 files changed, 127 insertions(+), 45 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3f26f2487d70cc8..7b687d311359c38 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -3,6 +3,7 @@ #endif PyAPI_FUNC(void) _Py_NewReference(PyObject *op); +PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); #ifdef Py_TRACE_REFS /* Py_TRACE_REFS is such major surgery that we call external routines. */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8796dfe2f6b8cf6..e15685f174ebcfe 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -37,11 +37,23 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( #define _Py_FatalRefcountError(message) \ _Py_FatalRefcountErrorFunc(__func__, (message)) + +#ifdef Py_REF_DEBUG +/* The symbol is only exposed in the API for the sake of extensions + built against the pre-3.12 stable ABI. */ +PyAPI_DATA(Py_ssize_t) _Py_RefTotal; + +extern void _Py_AddRefTotal(Py_ssize_t); +extern void _Py_IncRefTotal(void); +extern void _Py_DecRefTotal(void); +# define _Py_DEC_REFTOTAL() _Py_RefTotal-- +#endif + // Increment reference count by n static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { #ifdef Py_REF_DEBUG - _Py_RefTotal += n; + _Py_AddRefTotal(n); #endif op->ob_refcnt += n; } @@ -52,7 +64,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -70,7 +82,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) { _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -80,6 +92,11 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) #endif } +#ifdef Py_REF_DEBUG +# undef _Py_DEC_REFTOTAL +#endif + + PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); diff --git a/Include/object.h b/Include/object.h index 3774f1267300054..844b9c4a51c3e47 100644 --- a/Include/object.h +++ b/Include/object.h @@ -490,7 +490,21 @@ you can count such references to the type object.) */ #ifdef Py_REF_DEBUG -PyAPI_DATA(Py_ssize_t) _Py_RefTotal; +# if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030A0000 +extern Py_ssize_t _Py_RefTotal; +# define _Py_INC_REFTOTAL() _Py_RefTotal++ +# define _Py_DEC_REFTOTAL() _Py_RefTotal-- +# elif defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +extern void _Py_IncRefTotal(void); +extern void _Py_DecRefTotal(void); +# define _Py_INC_REFTOTAL() _Py_IncRefTotal() +# define _Py_DEC_REFTOTAL() _Py_DecRefTotal() +# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030C0000 +extern void _Py_IncRefTotal_DO_NOT_USE_THIS(void); +extern void _Py_DecRefTotal_DO_NOT_USE_THIS(void); +# define _Py_INC_REFTOTAL() _Py_IncRefTotal_DO_NOT_USE_THIS() +# define _Py_DEC_REFTOTAL() _Py_DecRefTotal_DO_NOT_USE_THIS() +# endif PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op); #endif /* Py_REF_DEBUG */ @@ -519,8 +533,8 @@ static inline void Py_INCREF(PyObject *op) // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. #ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif + _Py_INC_REFTOTAL(); +#endif // Py_REF_DEBUG op->ob_refcnt++; #endif } @@ -539,7 +553,7 @@ static inline void Py_DECREF(PyObject *op) { static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { _Py_DECREF_STAT_INC(); - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); if (--op->ob_refcnt != 0) { if (op->ob_refcnt < 0) { _Py_NegativeRefcount(filename, lineno, op); @@ -564,6 +578,9 @@ static inline void Py_DECREF(PyObject *op) #define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) #endif +#undef _Py_INC_REFTOTAL +#undef _Py_DEC_REFTOTAL + /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 10e507d6b481de0..ea67017a1ba3b16 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1654,15 +1654,10 @@ slot_tp_del(PyObject *self) */ { Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); + _Py_NewReferenceNoTotal(self); Py_SET_REFCNT(self, refcnt); } assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); - /* If Py_REF_DEBUG macro is defined, _Py_NewReference() increased - _Py_RefTotal, so we need to undo that. */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif } static PyObject * diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 657443f31fa709b..687a654bdae137c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3060,21 +3060,20 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return 0; } - /* XXX UNREF/NEWREF interface should be more symmetrical */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif *pv = (PyObject *) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(); +#endif PyObject_Free(v); PyErr_NoMemory(); return -1; } - _Py_NewReference(*pv); + _Py_NewReferenceNoTotal(*pv); sv = (PyBytesObject *) *pv; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 75c92172a91778c..a60f275742a5341 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -303,7 +303,7 @@ static inline void dictkeys_incref(PyDictKeysObject *dk) { #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif dk->dk_refcnt++; } @@ -313,7 +313,7 @@ dictkeys_decref(PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { free_keys_object(dk); @@ -633,7 +633,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } } #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; @@ -821,7 +821,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif return keys; } @@ -1520,7 +1520,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) // We can not use free_keys_object here because key's reference // are moved already. #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif if (oldkeys == Py_EMPTY_KEYS) { oldkeys->dk_refcnt--; diff --git a/Objects/object.c b/Objects/object.c index 5db2b6af21ef135..38da4d497a96e72 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -56,6 +56,24 @@ _PyObject_CheckConsistency(PyObject *op, int check_content) #ifdef Py_REF_DEBUG Py_ssize_t _Py_RefTotal; +static inline void +reftotal_increment(void) +{ + _Py_RefTotal++; +} + +static inline void +reftotal_decrement(void) +{ + _Py_RefTotal--; +} + +void +_Py_AddRefTotal(Py_ssize_t n) +{ + _Py_RefTotal += n; +} + Py_ssize_t _Py_GetRefTotal(void) { @@ -121,6 +139,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) filename, lineno, __func__); } +/* This is exposed strictly for use in Py_INCREF(). */ +PyAPI_FUNC(void) +_Py_IncRefTotal_DO_NOT_USE_THIS(void) +{ + reftotal_increment(); +} + +/* This is exposed strictly for use in Py_DECREF(). */ +PyAPI_FUNC(void) +_Py_DecRefTotal_DO_NOT_USE_THIS(void) +{ + reftotal_decrement(); +} + +void +_Py_IncRefTotal(void) +{ + reftotal_increment(); +} + +void +_Py_DecRefTotal(void) +{ + reftotal_decrement(); +} + #endif /* Py_REF_DEBUG */ void @@ -138,12 +182,18 @@ Py_DecRef(PyObject *o) void _Py_IncRef(PyObject *o) { +#ifdef Py_REF_DEBUG + reftotal_increment(); +#endif Py_INCREF(o); } void _Py_DecRef(PyObject *o) { +#ifdef Py_REF_DEBUG + reftotal_decrement(); +#endif Py_DECREF(o); } @@ -238,17 +288,12 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* tp_finalize resurrected it! Make it look like the original Py_DECREF * never happened. */ Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); + _Py_NewReferenceNoTotal(self); Py_SET_REFCNT(self, refcnt); _PyObject_ASSERT(self, (!_PyType_IS_GC(Py_TYPE(self)) || _PyObject_GC_IS_TRACKED(self))); - /* If Py_REF_DEBUG macro is defined, _Py_NewReference() increased - _Py_RefTotal, so we need to undo that. */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif return -1; } @@ -2010,21 +2055,33 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) } -void -_Py_NewReference(PyObject *op) +static inline void +new_reference(PyObject *op) { if (_PyRuntime.tracemalloc.config.tracing) { _PyTraceMalloc_NewReference(op); } -#ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif Py_SET_REFCNT(op, 1); #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op, 1); #endif } +void +_Py_NewReference(PyObject *op) +{ +#ifdef Py_REF_DEBUG + reftotal_increment(); +#endif + new_reference(op); +} + +void +_Py_NewReferenceNoTotal(PyObject *op) +{ + new_reference(op); +} + #ifdef Py_TRACE_REFS void diff --git a/Objects/structseq.c b/Objects/structseq.c index 100ccfef0a23c41..c20962ecd82563c 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -592,7 +592,7 @@ _PyStructSequence_FiniType(PyTypeObject *type) // Don't use Py_DECREF(): static type must not be deallocated Py_SET_REFCNT(type, 0); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif // Make sure that _PyStructSequence_InitType() will initialize diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 6ee93ab5adc2816..59c0251639d3dda 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -930,10 +930,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return *pv == NULL ? -1 : 0; } - /* XXX UNREF/NEWREF interface should be more symmetrical */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif if (_PyObject_GC_IS_TRACKED(v)) { _PyObject_GC_UNTRACK(v); } @@ -947,10 +943,13 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) sv = PyObject_GC_Resize(PyTupleObject, v, newsize); if (sv == NULL) { *pv = NULL; +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(); +#endif PyObject_GC_Del(v); return -1; } - _Py_NewReference((PyObject *) sv); + _Py_NewReferenceNoTotal((PyObject *) sv); /* Zero out items added by growing */ if (newsize > oldsize) memset(&sv->ob_item[oldsize], 0, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1ba30421c66dba4..2d50f9c340f2f36 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -947,21 +947,18 @@ resize_compact(PyObject *unicode, Py_ssize_t length) _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); #endif new_unicode = (PyObject *)PyObject_Realloc(unicode, new_size); if (new_unicode == NULL) { - _Py_NewReference(unicode); + _Py_NewReferenceNoTotal(unicode); PyErr_NoMemory(); return NULL; } unicode = new_unicode; - _Py_NewReference(unicode); + _Py_NewReferenceNoTotal(unicode); _PyUnicode_LENGTH(unicode) = length; #ifdef Py_DEBUG From 66ff374d4f353ae427c148d2a1d141d223303a82 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 15:56:36 -0700 Subject: [PATCH 14/35] gh-100227: Move func_state.next_version to PyInterpreterState (gh-102334) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_interp.h | 1 + Include/internal/pycore_runtime.h | 2 -- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/funcobject.c | 5 +++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 5cedb33d7e3afdb..11988149843fefc 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -10,7 +10,7 @@ extern "C" { #define FUNC_MAX_WATCHERS 8 -struct _py_func_runtime_state { +struct _py_func_state { uint32_t next_version; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7ef9c40153e4211..9efed0a1cf90c27 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -141,6 +141,7 @@ struct _is { struct _Py_float_state float_state; struct _Py_long_state long_state; struct _dtoa_state dtoa; + struct _py_func_state func_state; /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ PySliceObject *slice_cache; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 2350eaab5976caf..e0e3d4ace0cfde1 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -13,7 +13,6 @@ extern "C" { #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_function.h" // struct _func_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState @@ -155,7 +154,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _Py_dict_runtime_state dict_state; - struct _py_func_runtime_state func_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a2cc7c87c2f3e29..aeabcfd2b9056ca 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -68,9 +68,6 @@ extern PyTypeObject _PyExc_MemoryError; .dict_state = { \ .next_keys_version = 2, \ }, \ - .func_state = { \ - .next_version = 1, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -116,6 +113,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .func_state = { \ + .next_version = 1, \ + }, \ .static_objects = { \ .singletons = { \ ._not_used = 1, \ diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 99048ea41c6c800..ce5d7bda32c0320 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -227,10 +227,11 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) if (func->vectorcall != _PyFunction_Vectorcall) { return 0; } - if (_PyRuntime.func_state.next_version == 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->func_state.next_version == 0) { return 0; } - uint32_t v = _PyRuntime.func_state.next_version++; + uint32_t v = interp->func_state.next_version++; func->func_version = v; return v; } From 5e5acd291f4387876afc641163e9f8ae5c65086c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 18:04:16 -0700 Subject: [PATCH 15/35] gh-100227: Move next_keys_version to PyInterpreterState (gh-102335) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 3 ++- Include/internal/pycore_dict_state.h | 4 +++- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/dictobject.c | 7 ++++--- Python/specialize.c | 16 +++++++++++----- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 1af5e59a677a9a1..12c3c708e29b955 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -37,7 +37,8 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys); +extern uint32_t _PyDictKeys_GetVersionForCurrentState( + PyInterpreterState *interp, PyDictKeysObject *dictkeys); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 77375ea8beb8777..c5142ee6763a86f 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -14,7 +14,6 @@ struct _Py_dict_runtime_state { * It is incremented each time that a dictionary is created and each * time that a dictionary is modified. */ uint64_t global_version; - uint32_t next_keys_version; }; @@ -30,6 +29,8 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + uint32_t next_keys_version; + #if PyDict_MAXFREELIST > 0 /* Dictionary reuse scheme to save calls to malloc and free */ PyDictObject *free_list[PyDict_MAXFREELIST]; @@ -37,6 +38,7 @@ struct _Py_dict_state { int numfree; int keys_numfree; #endif + PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index aeabcfd2b9056ca..efc82b43a61dab1 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -65,9 +65,6 @@ extern PyTypeObject _PyExc_MemoryError; .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ }, \ - .dict_state = { \ - .next_keys_version = 2, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -113,6 +110,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .dict_state = { \ + .next_keys_version = 2, \ + }, \ .func_state = { \ .next_version = 1, \ }, \ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a60f275742a5341..b58e93f4320d425 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5655,15 +5655,16 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys) dictkeys_decref(keys); } -uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys) +uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, + PyDictKeysObject *dictkeys) { if (dictkeys->dk_version != 0) { return dictkeys->dk_version; } - if (_PyRuntime.dict_state.next_keys_version == 0) { + if (interp->dict_state.next_keys_version == 0) { return 0; } - uint32_t v = _PyRuntime.dict_state.next_keys_version++; + uint32_t v = interp->dict_state.next_keys_version++; dictkeys->dk_version = v; return v; } diff --git a/Python/specialize.c b/Python/specialize.c index 3405d2b0ab06803..0a7af8991eec83c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -512,7 +512,8 @@ specialize_module_load_attr( SPEC_FAIL_OUT_OF_RANGE); return -1; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), dict->ma_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; @@ -1063,7 +1064,8 @@ PyObject *descr, DescriptorClassification kind) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); return 0; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; @@ -1134,12 +1136,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (index != DKIX_EMPTY) { if (index != (uint16_t)index) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1167,12 +1171,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (globals_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } - uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys); + uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( + interp, builtin_keys); if (builtins_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; From 58d761e5b5f253892a697941c71bac0159a1d74e Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Wed, 8 Mar 2023 18:39:33 -0800 Subject: [PATCH 16/35] GH-84783: Document GH-101264 (Make the slice object hashable) in What's New. (#102548) --- Doc/whatsnew/3.12.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c0c021c679147f8..d982cb62ec2f4e3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -189,6 +189,8 @@ Other Language Changes part of comprehensions (like ``a``) is still disallowed, as per :pep:`572`. (Contributed by Nikita Sobolev in :gh:`100581`.) +* :class:`slice` objects are now hashable, allowing them to be used as dict keys and + set items. (Contributed by Furkan Onder in :gh:`101264`.) New Modules =========== From b45d14b88611fefc6f054226d3e1117082d322c8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 08:16:30 -0700 Subject: [PATCH 17/35] gh-100227: Move dict_state.global_version to PyInterpreterState (gh-102338) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 11 +- Include/internal/pycore_dict_state.h | 12 +- Include/internal/pycore_runtime.h | 2 - Objects/dictobject.c | 236 +++++++++++++++++---------- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 4 +- 6 files changed, 160 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 12c3c708e29b955..6253e0841ad3492 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -149,8 +149,8 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) #define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) -#define DICT_NEXT_VERSION() \ - (_PyRuntime.dict_state.global_version += DICT_VERSION_INCREMENT) +#define DICT_NEXT_VERSION(INTERP) \ + ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) void _PyDict_SendEvent(int watcher_bits, @@ -160,7 +160,8 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline uint64_t -_PyDict_NotifyEvent(PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyInterpreterState *interp, + PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) @@ -169,9 +170,9 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION() | watcher_bits; + return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(); + return DICT_NEXT_VERSION(interp); } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index c5142ee6763a86f..d608088ed2d7cec 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -9,14 +9,6 @@ extern "C" { #endif -struct _Py_dict_runtime_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; -}; - - #ifndef WITH_FREELISTS // without freelists # define PyDict_MAXFREELIST 0 @@ -29,6 +21,10 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + /*Global counter used to set ma_version_tag field of dictionary. + * It is incremented each time that a dictionary is created and each + * time that a dictionary is modified. */ + uint64_t global_version; uint32_t next_keys_version; #if PyDict_MAXFREELIST > 0 diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e0e3d4ace0cfde1..520109ca440444b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,7 +10,6 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects @@ -153,7 +152,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; - struct _Py_dict_runtime_state dict_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b58e93f4320d425..227e438a8dfffce 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -232,7 +232,8 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); +static int dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyDictObject *dict); @@ -241,9 +242,8 @@ static PyObject* dict_iter(PyDictObject *dict); #if PyDict_MAXFREELIST > 0 static struct _Py_dict_state * -get_dict_state(void) +get_dict_state(PyInterpreterState *interp) { - PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->dict_state; } #endif @@ -289,7 +289,8 @@ void _PyDict_DebugMallocStats(FILE *out) { #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_dict_state *state = get_dict_state(interp); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); #endif @@ -297,7 +298,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); static inline void dictkeys_incref(PyDictKeysObject *dk) @@ -309,14 +310,14 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { - free_keys_object(dk); + free_keys_object(interp, dk); } } @@ -586,7 +587,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(uint8_t log2_size, bool unicode) +new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { PyDictKeysObject *dk; Py_ssize_t usable; @@ -612,7 +613,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -648,7 +649,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) +free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) { assert(keys != Py_EMPTY_KEYS); if (DK_IS_UNICODE(keys)) { @@ -668,7 +669,7 @@ free_keys_object(PyDictKeysObject *keys) } } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // free_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -709,12 +710,14 @@ free_values(PyDictValues *values) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) +new_dict(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values, + Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp; assert(keys != NULL); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -731,7 +734,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); if (free_values_on_failure) { free_values(values); } @@ -741,7 +744,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = DICT_NEXT_VERSION(interp); ASSERT_CONSISTENT(mp); return (PyObject *)mp; } @@ -754,19 +757,19 @@ shared_keys_usable_size(PyDictKeysObject *keys) /* Consumes a reference to the keys object */ static PyObject * -new_dict_with_shared_keys(PyDictKeysObject *keys) +new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(keys, values, 0, 1); + return new_dict(interp, keys, values, 0, 1); } @@ -829,8 +832,9 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); dictkeys_incref(Py_EMPTY_KEYS); - return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1170,9 +1174,9 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyDictObject *mp, int unicode) +insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) { - return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static Py_ssize_t @@ -1214,12 +1218,13 @@ Returns -1 if an error occurred, or 0 on success. Consumes key and value references. */ static int -insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) +insertdict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(mp, 0) < 0) + if (insertion_resize(interp, mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } @@ -1231,13 +1236,14 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp, 1) < 0) + if (insertion_resize(interp, mp, 1) < 0) goto Fail; } @@ -1275,7 +1281,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { mp->ma_values->values[ix] = value; if (old_value == NULL) { @@ -1308,21 +1315,23 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) // Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. // Consumes key and value references. static int -insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, - PyObject *value) +insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object( + interp, PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); return -1; } - dictkeys_decref(Py_EMPTY_KEYS); + dictkeys_decref(interp, Py_EMPTY_KEYS); mp->ma_keys = newkeys; mp->ma_values = NULL; @@ -1397,7 +1406,8 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) +dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys; PyDictValues *oldvalues; @@ -1421,7 +1431,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(log2_newsize, unicode); + mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); if (mp->ma_keys == NULL) { mp->ma_keys = oldkeys; return -1; @@ -1462,7 +1472,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } build_indices_unicode(mp->ma_keys, newentries, numentries); } - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); mp->ma_values = NULL; free_values(oldvalues); } @@ -1530,7 +1540,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // dictresize() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -1557,7 +1567,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } static PyObject * -dict_new_presized(Py_ssize_t minused, bool unicode) +dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -1578,16 +1588,17 @@ dict_new_presized(Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(log2_newsize, unicode); + new_keys = new_keys_object(interp, log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(new_keys, NULL, 0, 0); + return new_dict(interp, new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - return dict_new_presized(minused, false); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_new_presized(interp, minused, false); } PyObject * @@ -1597,6 +1608,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -1606,7 +1618,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(length, unicode); + PyObject *dict = dict_new_presized(interp, length, unicode); if (dict == NULL) { return NULL; } @@ -1834,11 +1846,12 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, key, hash, value); + return insert_to_emptydict(interp, mp, key, hash, value); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, key, hash, value); + return insertdict(interp, mp, key, hash, value); } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the @@ -1875,11 +1888,12 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } static void @@ -1977,7 +1991,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hash, ix, old_value, new_version); } @@ -2020,7 +2036,9 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, assert(hashpos >= 0); if (res > 0) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hashpos, ix, old_value, new_version); } else { return 0; @@ -2045,7 +2063,9 @@ PyDict_Clear(PyObject *op) return; } /* Empty the dict... */ - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); dictkeys_incref(Py_EMPTY_KEYS); mp->ma_keys = Py_EMPTY_KEYS; mp->ma_values = NULL; @@ -2057,11 +2077,11 @@ PyDict_Clear(PyObject *op) for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); free_values(oldvalues); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } ASSERT_CONSISTENT(mp); } @@ -2165,6 +2185,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d Py_ssize_t ix; PyObject *old_value; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(PyDict_Check(dict)); mp = (PyDictObject *)dict; @@ -2187,7 +2208,8 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d return NULL; } assert(old_value != NULL); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); ASSERT_CONSISTENT(mp); @@ -2222,6 +2244,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; PyObject *d; int status; + PyInterpreterState *interp = _PyInterpreterState_GET(); d = _PyObject_CallNoArgs(cls); if (d == NULL) @@ -2236,13 +2259,16 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_hash_t hash; int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(mp, estimate_log2_keysize(PyDict_GET_SIZE(iterable)), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + unicode)) { Py_DECREF(d); return NULL; } while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2255,13 +2281,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { Py_DECREF(d); return NULL; } while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2308,9 +2335,10 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); - _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); if (Py_REFCNT(mp) > 1) { Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); return; @@ -2328,14 +2356,14 @@ dict_dealloc(PyDictObject *mp) Py_XDECREF(values->values[i]); } free_values(values); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -2765,7 +2793,7 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) } static int -dict_merge(PyObject *a, PyObject *b, int override) +dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { PyDictObject *mp, *other; @@ -2799,13 +2827,14 @@ dict_merge(PyObject *a, PyObject *b, int override) other->ma_used == okeys->dk_nentries && (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, b, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, b, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) { return -1; } - dictkeys_decref(mp->ma_keys); + dictkeys_decref(interp, mp->ma_keys); mp->ma_keys = keys; if (mp->ma_values != NULL) { free_values(mp->ma_values); @@ -2830,7 +2859,9 @@ dict_merge(PyObject *a, PyObject *b, int override) */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { return -1; } } @@ -2845,12 +2876,14 @@ dict_merge(PyObject *a, PyObject *b, int override) Py_INCREF(key); Py_INCREF(value); if (override == 1) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else { err = _PyDict_Contains_KnownHash(a, key, hash); if (err == 0) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else if (err > 0) { if (override != 0) { @@ -2936,20 +2969,23 @@ dict_merge(PyObject *a, PyObject *b, int override) int PyDict_Update(PyObject *a, PyObject *b) { - return dict_merge(a, b, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, 1); } int PyDict_Merge(PyObject *a, PyObject *b, int override) { + PyInterpreterState *interp = _PyInterpreterState_GET(); /* XXX Deprecate override not in (0, 1). */ - return dict_merge(a, b, override != 0); + return dict_merge(interp, a, b, override != 0); } int _PyDict_MergeEx(PyObject *a, PyObject *b, int override) { - return dict_merge(a, b, override); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, override); } static PyObject * @@ -2963,6 +2999,7 @@ PyDict_Copy(PyObject *o) { PyObject *copy; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); @@ -2991,7 +3028,7 @@ PyDict_Copy(PyObject *o) split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(); + split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); for (size_t i = 0; i < size; i++) { PyObject *value = mp->ma_values->values[i]; @@ -3024,7 +3061,7 @@ PyDict_Copy(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -3044,7 +3081,7 @@ PyDict_Copy(PyObject *o) copy = PyDict_New(); if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1) == 0) + if (dict_merge(interp, copy, o, 1) == 0) return copy; Py_DECREF(copy); return NULL; @@ -3244,6 +3281,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyDict_Check(d)) { PyErr_BadInternalCall(); @@ -3257,7 +3295,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (mp->ma_keys == Py_EMPTY_KEYS) { - if (insert_to_emptydict(mp, Py_NewRef(key), hash, + if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(defaultobj)) < 0) { return NULL; } @@ -3265,7 +3303,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(mp, 0) < 0) { + if (insertion_resize(interp, mp, 0) < 0) { return NULL; } } @@ -3275,11 +3313,12 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); mp->ma_keys->dk_version = 0; value = defaultobj; if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(mp, 1) < 0) { + if (insertion_resize(interp, mp, 1) < 0) { return NULL; } } @@ -3314,7 +3353,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) assert(mp->ma_keys->dk_usable >= 0); } else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); value = defaultobj; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); @@ -3395,6 +3435,7 @@ dict_popitem_impl(PyDictObject *self) Py_ssize_t i, j; PyObject *res; uint64_t new_version; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which @@ -3415,7 +3456,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { Py_DECREF(res); return NULL; } @@ -3434,7 +3475,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3449,7 +3491,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3708,7 +3751,8 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION(); + d->ma_version_tag = DICT_NEXT_VERSION( + _PyInterpreterState_GET()); dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -5262,7 +5306,9 @@ dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(void) { - PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = new_keys_object( + interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -5306,6 +5352,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp) int _PyObject_InitializeDict(PyObject *obj) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (tp->tp_dictoffset == 0) { return 0; @@ -5317,7 +5364,7 @@ _PyObject_InitializeDict(PyObject *obj) PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); } else { dict = PyDict_New(); @@ -5331,7 +5378,8 @@ _PyObject_InitializeDict(PyObject *obj) } static PyObject * -make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -5344,7 +5392,7 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(keys, values, used, 0); + PyObject *res = new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } @@ -5354,15 +5402,17 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) PyObject * _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(keys, values); + return make_dict_from_instance_attributes(interp, keys, values); } int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); @@ -5385,7 +5435,8 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, OBJECT_STAT_INC(dict_materialized_str_subclass); } #endif - PyObject *dict = make_dict_from_instance_attributes(keys, values); + PyObject *dict = make_dict_from_instance_attributes( + interp, keys, values); if (dict == NULL) { return -1; } @@ -5564,13 +5615,15 @@ PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { PyObject *dict; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); if (_PyDictOrValues_IsValues(*dorv_ptr)) { PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), values); + dict = make_dict_from_instance_attributes( + interp, CACHED_KEYS(tp), values); if (dict != NULL) { dorv_ptr->dict = dict; } @@ -5579,7 +5632,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = _PyDictOrValues_GetDict(*dorv_ptr); if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); dorv_ptr->dict = dict; } } @@ -5596,7 +5649,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + *dictptr = dict = new_dict_with_shared_keys( + interp, CACHED_KEYS(tp)); } else { *dictptr = dict = PyDict_New(); @@ -5613,6 +5667,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *dict; int res; PyDictKeysObject *cached; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { @@ -5620,7 +5675,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = *dictptr; if (dict == NULL) { dictkeys_incref(cached); - dict = new_dict_with_shared_keys(cached); + dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; *dictptr = dict; @@ -5652,7 +5707,8 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - dictkeys_decref(keys); + PyInterpreterState *interp = _PyInterpreterState_GET(); + dictkeys_decref(interp, keys); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 63dbecad3b45fc8..45d507266743211 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1679,7 +1679,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -1687,7 +1687,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 82e18505b0d4306..51357cda4c8949c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2114,7 +2114,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -2122,7 +2122,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); From cf6e7c5e551b3513817d6a77ba88253dc8473298 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:46:21 -0700 Subject: [PATCH 18/35] gh-100227: Isolate the Import State to Each Interpreter (gh-101941) Specific changes: * move the import lock to PyInterpreterState * move the "find_and_load" diagnostic state to PyInterpreterState Note that the import lock exists to keep multiple imports of the same module in the same interpreter (but in different threads) from stomping on each other. Independently, we use a distinct global lock to protect globally shared import state, especially related to loaded extension modules. For now we can rely on the GIL as that lock but with a per-interpreter GIL we'll need a new global lock. The remaining state in _PyRuntimeState.imports will (probably) continue being global. https://github.com/python/cpython/issues/100227 --- Include/cpython/import.h | 4 +- Include/internal/pycore_import.h | 34 +++++---- Include/internal/pycore_runtime_init.h | 10 --- Modules/posixmodule.c | 13 ++-- Python/import.c | 98 ++++++++++++++------------ 5 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index a58801b47f1bec2..2bca4ade4c4f2cf 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,8 +10,8 @@ PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name); PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); -PyAPI_FUNC(void) _PyImport_AcquireLock(void); -PyAPI_FUNC(int) _PyImport_ReleaseLock(void); +PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp); +PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index b7ffe01c0c0e207..69ed6273b7e6091 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -21,17 +21,6 @@ struct _import_runtime_state { This is initialized lazily in _PyImport_FixupExtensionObject(). Modules are added there and looked up in _imp.find_extension(). */ PyObject *extensions; - /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; - struct { - int import_level; - _PyTime_t accumulated; - int header; - } find_and_load; /* Package context -- the full module name for package imports */ const char * pkgcontext; }; @@ -69,6 +58,18 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + /* The global import lock. */ + struct { + PyThread_type_lock mutex; + unsigned long thread; + int level; + } lock; + /* diagnostic info in PyImport_ImportModuleLevelObject() */ + struct { + int import_level; + _PyTime_t accumulated; + int header; + } find_and_load; }; #ifdef HAVE_DLOPEN @@ -86,8 +87,15 @@ struct _import_state { #define IMPORTS_INIT \ { \ - .override_frozen_modules = 0, \ DLOPENFLAGS_INIT \ + .lock = { \ + .mutex = NULL, \ + .thread = PYTHREAD_INVALID_THREAD_ID, \ + .level = 0, \ + }, \ + .find_and_load = { \ + .header = 1, \ + }, \ } extern void _PyImport_ClearCore(PyInterpreterState *interp); @@ -138,7 +146,7 @@ extern void _PyImport_FiniExternal(PyInterpreterState *interp); #ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(void); +extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index efc82b43a61dab1..bdecac944dfd3a2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -40,16 +40,6 @@ extern PyTypeObject _PyExc_MemoryError; in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ - .imports = { \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ - .find_and_load = { \ - .header = 1, \ - }, \ - }, \ .ceval = { \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7beb2cee64a05c6..a3d86cbe7a57fb2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -567,18 +567,21 @@ run_at_forkers(PyObject *lst, int reverse) void PyOS_BeforeFork(void) { - run_at_forkers(_PyInterpreterState_GET()->before_forkers, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + run_at_forkers(interp->before_forkers, 1); - _PyImport_AcquireLock(); + _PyImport_AcquireLock(interp); } void PyOS_AfterFork_Parent(void) { - if (_PyImport_ReleaseLock() <= 0) + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) <= 0) { Py_FatalError("failed releasing import lock after fork"); + } - run_at_forkers(_PyInterpreterState_GET()->after_forkers_parent, 0); + run_at_forkers(interp->after_forkers_parent, 0); } void @@ -604,7 +607,7 @@ PyOS_AfterFork_Child(void) goto fatal_error; } - status = _PyImport_ReInitLock(); + status = _PyImport_ReInitLock(tstate->interp); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } diff --git a/Python/import.c b/Python/import.c index 57d4eea148810f8..1bf4199e125aa5d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -56,11 +56,6 @@ static struct _inittab *inittab_copy = NULL; #define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index #define EXTENSIONS _PyRuntime.imports.extensions -#define import_lock _PyRuntime.imports.lock.mutex -#define import_lock_thread _PyRuntime.imports.lock.thread -#define import_lock_level _PyRuntime.imports.lock.level - -#define FIND_AND_LOAD _PyRuntime.imports.find_and_load #define PKGCONTEXT (_PyRuntime.imports.pkgcontext) @@ -85,6 +80,16 @@ static struct _inittab *inittab_copy = NULL; #define IMPORT_FUNC(interp) \ (interp)->imports.import_func +#define IMPORT_LOCK(interp) \ + (interp)->imports.lock.mutex +#define IMPORT_LOCK_THREAD(interp) \ + (interp)->imports.lock.thread +#define IMPORT_LOCK_LEVEL(interp) \ + (interp)->imports.lock.level + +#define FIND_AND_LOAD(interp) \ + (interp)->imports.find_and_load + /*******************/ /* the import lock */ @@ -95,45 +100,45 @@ static struct _inittab *inittab_copy = NULL; These calls are serialized by the global interpreter lock. */ void -_PyImport_AcquireLock(void) +_PyImport_AcquireLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); if (me == PYTHREAD_INVALID_THREAD_ID) return; /* Too bad */ - if (import_lock == NULL) { - import_lock = PyThread_allocate_lock(); - if (import_lock == NULL) + if (IMPORT_LOCK(interp) == NULL) { + IMPORT_LOCK(interp) = PyThread_allocate_lock(); + if (IMPORT_LOCK(interp) == NULL) return; /* Nothing much we can do. */ } - if (import_lock_thread == me) { - import_lock_level++; + if (IMPORT_LOCK_THREAD(interp) == me) { + IMPORT_LOCK_LEVEL(interp)++; return; } - if (import_lock_thread != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(import_lock, 0)) + if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || + !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) { PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); PyEval_RestoreThread(tstate); } - assert(import_lock_level == 0); - import_lock_thread = me; - import_lock_level = 1; + assert(IMPORT_LOCK_LEVEL(interp) == 0); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp) = 1; } int -_PyImport_ReleaseLock(void) +_PyImport_ReleaseLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || import_lock == NULL) + if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) return 0; /* Too bad */ - if (import_lock_thread != me) + if (IMPORT_LOCK_THREAD(interp) != me) return -1; - import_lock_level--; - assert(import_lock_level >= 0); - if (import_lock_level == 0) { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(import_lock); + IMPORT_LOCK_LEVEL(interp)--; + assert(IMPORT_LOCK_LEVEL(interp) >= 0); + if (IMPORT_LOCK_LEVEL(interp) == 0) { + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + PyThread_release_lock(IMPORT_LOCK(interp)); } return 1; } @@ -144,23 +149,23 @@ _PyImport_ReleaseLock(void) We now acquire the import lock around fork() calls but on some platforms (Solaris 9 and earlier? see isue7242) that still left us with problems. */ PyStatus -_PyImport_ReInitLock(void) +_PyImport_ReInitLock(PyInterpreterState *interp) { - if (import_lock != NULL) { - if (_PyThread_at_fork_reinit(&import_lock) < 0) { + if (IMPORT_LOCK(interp) != NULL) { + if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { return _PyStatus_ERR("failed to create a new lock"); } } - if (import_lock_level > 1) { + if (IMPORT_LOCK_LEVEL(interp) > 1) { /* Forked as a side effect of import */ unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); - import_lock_thread = me; - import_lock_level--; + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp)--; } else { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - import_lock_level = 0; + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + IMPORT_LOCK_LEVEL(interp) = 0; } return _PyStatus_OK(); } @@ -2506,8 +2511,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level FIND_AND_LOAD.import_level -#define accumulated FIND_AND_LOAD.accumulated +#define import_level FIND_AND_LOAD(interp).import_level +#define accumulated FIND_AND_LOAD(interp).accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -2528,7 +2533,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header FIND_AND_LOAD.header +#define header FIND_AND_LOAD(interp).header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -2867,10 +2872,6 @@ _PyImport_Fini(void) { /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ _extensions_cache_clear_all(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } /* Use the same memory allocator as _PyImport_Init(). */ PyMemAllocatorEx old_alloc; @@ -2959,6 +2960,11 @@ _PyImport_FiniCore(PyInterpreterState *interp) PyErr_WriteUnraisable(NULL); } + if (IMPORT_LOCK(interp) != NULL) { + PyThread_free_lock(IMPORT_LOCK(interp)); + IMPORT_LOCK(interp) = NULL; + } + _PyImport_ClearCore(interp); } @@ -3090,7 +3096,9 @@ static PyObject * _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyBool_FromLong( + IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); } /*[clinic input] @@ -3106,7 +3114,8 @@ static PyObject * _imp_acquire_lock_impl(PyObject *module) /*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ { - _PyImport_AcquireLock(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_AcquireLock(interp); Py_RETURN_NONE; } @@ -3122,7 +3131,8 @@ static PyObject * _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { - if (_PyImport_ReleaseLock() < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) < 0) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; From ca066bdbed85094a9c4d9930823ce3587807db48 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:50:33 -0700 Subject: [PATCH 19/35] gh-81057: Vendor a Subset of distutils for the c-analyzer Tool (gh-102505) distutils was removed in November. However, the c-analyzer relies on it. To solve that here, we vendor the parts the tool needs so it can be run against 3.12+. (Also see gh-92584.) Note that we may end up removing this code later in favor of a solution in common with the peg_generator tool (which also relies on distutils). At the least, the copy here makes sure the c-analyzer tool works on 3.12+ in the meantime. --- Tools/c-analyzer/distutils/README | 2 + Tools/c-analyzer/distutils/__init__.py | 0 Tools/c-analyzer/distutils/_msvccompiler.py | 203 ++++++++ Tools/c-analyzer/distutils/bcppcompiler.py | 109 ++++ Tools/c-analyzer/distutils/ccompiler.py | 470 ++++++++++++++++++ Tools/c-analyzer/distutils/cygwinccompiler.py | 286 +++++++++++ Tools/c-analyzer/distutils/debug.py | 5 + Tools/c-analyzer/distutils/dep_util.py | 29 ++ Tools/c-analyzer/distutils/errors.py | 48 ++ Tools/c-analyzer/distutils/log.py | 63 +++ Tools/c-analyzer/distutils/msvc9compiler.py | 438 ++++++++++++++++ Tools/c-analyzer/distutils/msvccompiler.py | 327 ++++++++++++ Tools/c-analyzer/distutils/spawn.py | 48 ++ Tools/c-analyzer/distutils/unixccompiler.py | 102 ++++ Tools/c-analyzer/distutils/util.py | 171 +++++++ 15 files changed, 2301 insertions(+) create mode 100644 Tools/c-analyzer/distutils/README create mode 100644 Tools/c-analyzer/distutils/__init__.py create mode 100644 Tools/c-analyzer/distutils/_msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/bcppcompiler.py create mode 100644 Tools/c-analyzer/distutils/ccompiler.py create mode 100644 Tools/c-analyzer/distutils/cygwinccompiler.py create mode 100644 Tools/c-analyzer/distutils/debug.py create mode 100644 Tools/c-analyzer/distutils/dep_util.py create mode 100644 Tools/c-analyzer/distutils/errors.py create mode 100644 Tools/c-analyzer/distutils/log.py create mode 100644 Tools/c-analyzer/distutils/msvc9compiler.py create mode 100644 Tools/c-analyzer/distutils/msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/spawn.py create mode 100644 Tools/c-analyzer/distutils/unixccompiler.py create mode 100644 Tools/c-analyzer/distutils/util.py diff --git a/Tools/c-analyzer/distutils/README b/Tools/c-analyzer/distutils/README new file mode 100644 index 000000000000000..b260b8e06fac06d --- /dev/null +++ b/Tools/c-analyzer/distutils/README @@ -0,0 +1,2 @@ +This is a partial copy of distutils as it was removed in 0faa0ba240e. +It only includes the parts needed by the C parser. diff --git a/Tools/c-analyzer/distutils/__init__.py b/Tools/c-analyzer/distutils/__init__.py new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/Tools/c-analyzer/distutils/_msvccompiler.py b/Tools/c-analyzer/distutils/_msvccompiler.py new file mode 100644 index 000000000000000..1e67870d13323d3 --- /dev/null +++ b/Tools/c-analyzer/distutils/_msvccompiler.py @@ -0,0 +1,203 @@ +"""distutils._msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for Microsoft Visual Studio 2015. + +The module is compatible with VS 2015 and later. You can find legacy support +for older versions in distutils.msvc9compiler and distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS 2005 and VS 2008 by Christian Heimes +# ported to VS 2015 by Steve Dower + +import os +import subprocess +import winreg + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +from itertools import count + +def _find_vc2015(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\VisualStudio\SxS\VC7", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY + ) + except OSError: + log.debug("Visual C++ is not registered") + return None, None + + best_version = 0 + best_dir = None + with key: + for i in count(): + try: + v, vc_dir, vt = winreg.EnumValue(key, i) + except OSError: + break + if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): + try: + version = int(float(v)) + except (ValueError, TypeError): + continue + if version >= 14 and version > best_version: + best_version, best_dir = version, vc_dir + return best_version, best_dir + +def _find_vc2017(): + """Returns "15, path" based on the result of invoking vswhere.exe + If no install is found, returns "None, None" + + The version is returned to avoid unnecessarily changing the function + result. It may be ignored when the path is not None. + + If vswhere.exe is not available, by definition, VS 2017 is not + installed. + """ + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + return None, None + + try: + path = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ], encoding="mbcs", errors="strict").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + return None, None + + path = os.path.join(path, "VC", "Auxiliary", "Build") + if os.path.isdir(path): + return 15, path + + return None, None + +PLAT_SPEC_TO_RUNTIME = { + 'x86' : 'x86', + 'x86_amd64' : 'x64', + 'x86_arm' : 'arm', + 'x86_arm64' : 'arm64' +} + +def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value + _, best_dir = _find_vc2017() + + if not best_dir: + best_version, best_dir = _find_vc2015() + + if not best_dir: + log.debug("No suitable Visual C++ version found") + return None, None + + vcvarsall = os.path.join(best_dir, "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + log.debug("%s cannot be found", vcvarsall) + return None, None + + return vcvarsall, None + +def _get_vc_env(plat_spec): + if os.getenv("DISTUTILS_USE_SDK"): + return { + key.lower(): value + for key, value in os.environ.items() + } + + vcvarsall, _ = _find_vcvarsall(plat_spec) + if not vcvarsall: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + + try: + out = subprocess.check_output( + 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), + stderr=subprocess.STDOUT, + ).decode('utf-16le', errors='replace') + except subprocess.CalledProcessError as exc: + log.error(exc.output) + raise DistutilsPlatformError("Error executing {}" + .format(exc.cmd)) + + env = { + key.lower(): value + for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value + } + + return env + +def _find_exe(exe, paths=None): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + if not paths: + paths = os.getenv('path').split(os.pathsep) + for p in paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + return exe + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Always cross-compile from x86 to work with the +# lighter-weight MSVC installs that do not include native 64-bit tools. +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'x86_amd64', + 'win-arm32' : 'x86_arm', + 'win-arm64' : 'x86_arm64' +} + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.initialized = False diff --git a/Tools/c-analyzer/distutils/bcppcompiler.py b/Tools/c-analyzer/distutils/bcppcompiler.py new file mode 100644 index 000000000000000..4575b665c10c0e5 --- /dev/null +++ b/Tools/c-analyzer/distutils/bcppcompiler.py @@ -0,0 +1,109 @@ +"""distutils.bcppcompiler + +Contains BorlandCCompiler, an implementation of the abstract CCompiler class +for the Borland C++ compiler. +""" + +# This implementation by Lyle Johnson, based on the original msvccompiler.py +# module and using the directions originally published by Gordon Williams. + +# XXX looks like there's a LOT of overlap between these two classes: +# someone should sit down and factor out the common code as +# WindowsCCompiler! --GPW + + +import os +from distutils.errors import DistutilsExecError, CompileError +from distutils.ccompiler import \ + CCompiler, gen_preprocess_options +from distutils.dep_util import newer + +class BCPPCompiler(CCompiler) : + """Concrete class that implements an interface to the Borland C/C++ + compiler, as defined by the CCompiler abstract class. + """ + + compiler_type = 'bcpp' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = _c_extensions + _cpp_extensions + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__ (self, + verbose=0, + dry_run=0, + force=0): + + CCompiler.__init__ (self, verbose, dry_run, force) + + # These executables are assumed to all be in the path. + # Borland doesn't seem to use any special registry settings to + # indicate their installation locations. + + self.cc = "bcc32.exe" + self.linker = "ilink32.exe" + self.lib = "tlib.exe" + + self.preprocess_options = None + self.compile_options = ['/tWM', '/O2', '/q', '/g0'] + self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] + + self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_static = [] + self.ldflags_exe = ['/Gn', '/q', '/x'] + self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] + + + # -- Worker methods ------------------------------------------------ + + def preprocess (self, + source, + output_file=None, + macros=None, + include_dirs=None, + extra_preargs=None, + extra_postargs=None): + + (_, macros, include_dirs) = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = ['cpp32.exe'] + pp_opts + if output_file is not None: + pp_args.append('-o' + output_file) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or the + # source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + print(msg) + raise CompileError(msg) + + # preprocess() diff --git a/Tools/c-analyzer/distutils/ccompiler.py b/Tools/c-analyzer/distutils/ccompiler.py new file mode 100644 index 000000000000000..13e43103b94f5eb --- /dev/null +++ b/Tools/c-analyzer/distutils/ccompiler.py @@ -0,0 +1,470 @@ +"""distutils.ccompiler + +Contains CCompiler, an abstract base class that defines the interface +for the Distutils compiler abstraction model.""" + +import sys, os, re +from distutils.errors import ( + DistutilsModuleError, DistutilsPlatformError, +) +from distutils.util import split_quoted + +class CCompiler: + """Abstract base class to define the interface that must be implemented + by real compiler classes. Also has some utility methods used by + several compiler classes. + + The basic idea behind a compiler abstraction class is that each + instance can be used for all the compile/link steps in building a + single project. Thus, attributes common to all of those compile and + link steps -- include directories, macros to define, libraries to link + against, etc. -- are attributes of the compiler instance. To allow for + variability in how individual files are treated, most of those + attributes may be varied on a per-compilation or per-link basis. + """ + + # 'compiler_type' is a class attribute that identifies this class. It + # keeps code that wants to know what kind of compiler it's dealing with + # from having to import all possible compiler classes just to do an + # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' + # should really, really be one of the keys of the 'compiler_class' + # dictionary (see below -- used by the 'new_compiler()' factory + # function) -- authors of new compiler interface classes are + # responsible for updating 'compiler_class'! + compiler_type = None + + # XXX things not handled by this compiler abstraction model: + # * client can't provide additional options for a compiler, + # e.g. warning, optimization, debugging flags. Perhaps this + # should be the domain of concrete compiler abstraction classes + # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base + # class should have methods for the common ones. + # * can't completely override the include or library searchg + # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". + # I'm not sure how widely supported this is even by Unix + # compilers, much less on other platforms. And I'm even less + # sure how useful it is; maybe for cross-compiling, but + # support for that is a ways off. (And anyways, cross + # compilers probably have a dedicated binary with the + # right paths compiled in. I hope.) + # * can't do really freaky things with the library list/library + # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against + # different versions of libfoo.a in different locations. I + # think this is useless without the ability to null out the + # library search path anyways. + + + # Subclasses that rely on the standard filename generation methods + # implemented below should override these; see the comment near + # those methods ('object_filenames()' et. al.) for details: + src_extensions = None # list of strings + obj_extension = None # string + static_lib_extension = None + shared_lib_extension = None # string + static_lib_format = None # format string + shared_lib_format = None # prob. same as static_lib_format + exe_extension = None # string + + # Default language settings. language_map is used to detect a source + # file or Extension target language, checking source filenames. + # language_order is used to detect the language precedence, when deciding + # what language to use when mixing source types. For example, if some + # extension has two files with ".c" extension, and one with ".cpp", it + # is still linked as c++. + language_map = {".c" : "c", + ".cc" : "c++", + ".cpp" : "c++", + ".cxx" : "c++", + ".m" : "objc", + } + language_order = ["c++", "objc", "c"] + + def __init__(self, verbose=0, dry_run=0, force=0): + self.dry_run = dry_run + self.force = force + self.verbose = verbose + + # 'output_dir': a common output directory for object, library, + # shared object, and shared library files + self.output_dir = None + + # 'macros': a list of macro definitions (or undefinitions). A + # macro definition is a 2-tuple (name, value), where the value is + # either a string or None (no explicit value). A macro + # undefinition is a 1-tuple (name,). + self.macros = [] + + # 'include_dirs': a list of directories to search for include files + self.include_dirs = [] + + # 'libraries': a list of libraries to include in any link + # (library names, not filenames: eg. "foo" not "libfoo.a") + self.libraries = [] + + # 'library_dirs': a list of directories to search for libraries + self.library_dirs = [] + + # 'runtime_library_dirs': a list of directories to search for + # shared libraries/objects at runtime + self.runtime_library_dirs = [] + + # 'objects': a list of object files (or similar, such as explicitly + # named library files) to include on any link + self.objects = [] + + for key in self.executables.keys(): + self.set_executable(key, self.executables[key]) + + def set_executables(self, **kwargs): + """Define the executables (and options for them) that will be run + to perform the various stages of compilation. The exact set of + executables that may be specified here depends on the compiler + class (via the 'executables' class attribute), but most will have: + compiler the C/C++ compiler + linker_so linker used to create shared objects and libraries + linker_exe linker used to create binary executables + archiver static library creator + + On platforms with a command-line (Unix, DOS/Windows), each of these + is a string that will be split into executable name and (optional) + list of arguments. (Splitting the string is done similarly to how + Unix shells operate: words are delimited by spaces, but quotes and + backslashes can override this. See + 'distutils.util.split_quoted()'.) + """ + + # Note that some CCompiler implementation classes will define class + # attributes 'cpp', 'cc', etc. with hard-coded executable names; + # this is appropriate when a compiler class is for exactly one + # compiler/OS combination (eg. MSVCCompiler). Other compiler + # classes (UnixCCompiler, in particular) are driven by information + # discovered at run-time, since there are many different ways to do + # basically the same things with Unix C compilers. + + for key in kwargs: + if key not in self.executables: + raise ValueError("unknown executable '%s' for class %s" % + (key, self.__class__.__name__)) + self.set_executable(key, kwargs[key]) + + def set_executable(self, key, value): + if isinstance(value, str): + setattr(self, key, split_quoted(value)) + else: + setattr(self, key, value) + + def _find_macro(self, name): + i = 0 + for defn in self.macros: + if defn[0] == name: + return i + i += 1 + return None + + def _check_macro_definitions(self, definitions): + """Ensures that every element of 'definitions' is a valid macro + definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do + nothing if all definitions are OK, raise TypeError otherwise. + """ + for defn in definitions: + if not (isinstance(defn, tuple) and + (len(defn) in (1, 2) and + (isinstance (defn[1], str) or defn[1] is None)) and + isinstance (defn[0], str)): + raise TypeError(("invalid macro definition '%s': " % defn) + \ + "must be tuple (string,), (string, string), or " + \ + "(string, None)") + + + # -- Bookkeeping methods ------------------------------------------- + + def define_macro(self, name, value=None): + """Define a preprocessor macro for all compilations driven by this + compiler object. The optional parameter 'value' should be a + string; if it is not supplied, then the macro will be defined + without an explicit value and the exact outcome depends on the + compiler used (XXX true? does ANSI say anything about this?) + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + self.macros.append((name, value)) + + def undefine_macro(self, name): + """Undefine a preprocessor macro for all compilations driven by + this compiler object. If the same macro is defined by + 'define_macro()' and undefined by 'undefine_macro()' the last call + takes precedence (including multiple redefinitions or + undefinitions). If the macro is redefined/undefined on a + per-compilation basis (ie. in the call to 'compile()'), then that + takes precedence. + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + undefn = (name,) + self.macros.append(undefn) + + def add_include_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + header files. The compiler is instructed to search directories in + the order in which they are supplied by successive calls to + 'add_include_dir()'. + """ + self.include_dirs.append(dir) + + def set_include_dirs(self, dirs): + """Set the list of directories that will be searched to 'dirs' (a + list of strings). Overrides any preceding calls to + 'add_include_dir()'; subsequence calls to 'add_include_dir()' add + to the list passed to 'set_include_dirs()'. This does not affect + any list of standard include directories that the compiler may + search by default. + """ + self.include_dirs = dirs[:] + + + # -- Private utility methods -------------------------------------- + # (here for the convenience of subclasses) + + # Helper method to prep compiler in subclass compile() methods + + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Typecheck and fix-up some of the arguments to the 'compile()' + method, and return fixed-up values. Specifically: if 'output_dir' + is None, replaces it with 'self.output_dir'; ensures that 'macros' + is a list, and augments it with 'self.macros'; ensures that + 'include_dirs' is a list, and augments it with 'self.include_dirs'. + Guarantees that the returned values are of the correct type, + i.e. for 'output_dir' either string or None, and for 'macros' and + 'include_dirs' either list or None. + """ + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if include_dirs is None: + include_dirs = self.include_dirs + elif isinstance(include_dirs, (list, tuple)): + include_dirs = list(include_dirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + return output_dir, macros, include_dirs + + + # -- Worker methods ------------------------------------------------ + # (must be implemented by subclasses) + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + """Preprocess a single C/C++ source file, named in 'source'. + Output will be written to file named 'output_file', or stdout if + 'output_file' not supplied. 'macros' is a list of macro + definitions as for 'compile()', which will augment the macros set + with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a + list of directory names that will be added to the default list. + + Raises PreprocessError on failure. + """ + pass + + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function; there is + # no appropriate default implementation so subclasses should + # implement all of these. + +# def library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for libraries. +# """ +# raise NotImplementedError +# +# def runtime_library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for runtime libraries. +# """ +# raise NotImplementedError +# +# def library_option(self, lib): +# """Return the compiler option to add 'lib' to the list of libraries +# linked into the shared library or executable. +# """ +# raise NotImplementedError +# +# def find_library_file (self, dirs, lib, debug=0): +# """Search the specified list of directories for a static or shared +# library file 'lib' and return the full path to that file. If +# 'debug' true, look for a debugging version (if that makes sense on +# the current platform). Return None if 'lib' wasn't found in any of +# the specified directories. +# """ +# raise NotImplementedError + + + # -- Utility methods ----------------------------------------------- + + def spawn(self, cmd): + raise NotImplementedError + + +# Map a sys.platform/os.name ('posix', 'nt') to the default compiler +# type for that platform. Keys are interpreted as re match +# patterns. Order is important; platform mappings are preferred over +# OS names. +_default_compilers = ( + + # Platform string mappings + + # on a cygwin built python we can use gcc like an ordinary UNIXish + # compiler + ('cygwin.*', 'unix'), + + # OS name mappings + ('posix', 'unix'), + ('nt', 'msvc'), + + ) + +def get_default_compiler(osname=None, platform=None): + """Determine the default compiler to use for the given platform. + + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + + The default values are os.name and sys.platform in case the + parameters are not given. + """ + if osname is None: + osname = os.name + if platform is None: + platform = sys.platform + for pattern, compiler in _default_compilers: + if re.match(pattern, platform) is not None or \ + re.match(pattern, osname) is not None: + return compiler + # Default to Unix compiler + return 'unix' + +# Map compiler types to (module_name, class_name) pairs -- ie. where to +# find the code that implements an interface to this compiler. (The module +# is assumed to be in the 'distutils' package.) +compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', + "standard UNIX-style compiler"), + 'msvc': ('_msvccompiler', 'MSVCCompiler', + "Microsoft Visual C++"), + 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', + "Cygwin port of GNU C Compiler for Win32"), + 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', + "Mingw32 port of GNU C Compiler for Win32"), + 'bcpp': ('bcppcompiler', 'BCPPCompiler', + "Borland C++ Compiler"), + } + + +def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): + """Generate an instance of some CCompiler subclass for the supplied + platform/compiler combination. 'plat' defaults to 'os.name' + (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler + for that platform. Currently only 'posix' and 'nt' are supported, and + the default compilers are "traditional Unix interface" (UnixCCompiler + class) and Visual C++ (MSVCCompiler class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a + Microsoft compiler object under Unix -- if you supply a value for + 'compiler', 'plat' is ignored. + """ + if plat is None: + plat = os.name + + try: + if compiler is None: + compiler = get_default_compiler(plat) + + (module_name, class_name, long_description) = compiler_class[compiler] + except KeyError: + msg = "don't know how to compile C/C++ code on platform '%s'" % plat + if compiler is not None: + msg = msg + " with '%s' compiler" % compiler + raise DistutilsPlatformError(msg) + + try: + module_name = "distutils." + module_name + __import__ (module_name) + module = sys.modules[module_name] + klass = vars(module)[class_name] + except ImportError: + raise + raise DistutilsModuleError( + "can't compile C/C++ code: unable to load module '%s'" % \ + module_name) + except KeyError: + raise DistutilsModuleError( + "can't compile C/C++ code: unable to find class '%s' " + "in module '%s'" % (class_name, module_name)) + + # XXX The None is necessary to preserve backwards compatibility + # with classes that expect verbose to be the first positional + # argument. + return klass(None, dry_run, force) + + +def gen_preprocess_options(macros, include_dirs): + """Generate C pre-processor options (-D, -U, -I) as used by at least + two types of compilers: the typical Unix compiler and Visual C++. + 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) + means undefine (-U) macro 'name', and (name,value) means define (-D) + macro 'name' to 'value'. 'include_dirs' is just a list of directory + names to be added to the header file search path (-I). Returns a list + of command-line options suitable for either Unix compilers or Visual + C++. + """ + # XXX it would be nice (mainly aesthetic, and so we don't generate + # stupid-looking command lines) to go over 'macros' and eliminate + # redundant definitions/undefinitions (ie. ensure that only the + # latest mention of a particular macro winds up on the command + # line). I don't think it's essential, though, since most (all?) + # Unix C compilers only pay attention to the latest -D or -U + # mention of a macro on their command line. Similar situation for + # 'include_dirs'. I'm punting on both for now. Anyways, weeding out + # redundancies like this should probably be the province of + # CCompiler, since the data structures used are inherited from it + # and therefore common to all CCompiler classes. + pp_opts = [] + for macro in macros: + if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): + raise TypeError( + "bad macro definition '%s': " + "each element of 'macros' list must be a 1- or 2-tuple" + % macro) + + if len(macro) == 1: # undefine this macro + pp_opts.append("-U%s" % macro[0]) + elif len(macro) == 2: + if macro[1] is None: # define with no explicit value + pp_opts.append("-D%s" % macro[0]) + else: + # XXX *don't* need to be clever about quoting the + # macro value here, because we're going to avoid the + # shell at all costs when we spawn the command! + pp_opts.append("-D%s=%s" % macro) + + for dir in include_dirs: + pp_opts.append("-I%s" % dir) + return pp_opts diff --git a/Tools/c-analyzer/distutils/cygwinccompiler.py b/Tools/c-analyzer/distutils/cygwinccompiler.py new file mode 100644 index 000000000000000..a3505f31f1f0a26 --- /dev/null +++ b/Tools/c-analyzer/distutils/cygwinccompiler.py @@ -0,0 +1,286 @@ +"""distutils.cygwinccompiler + +Provides the CygwinCCompiler class, a subclass of UnixCCompiler that +handles the Cygwin port of the GNU C compiler to Windows. It also contains +the Mingw32CCompiler class which handles the mingw32 port of GCC (same as +cygwin in no-cygwin mode). +""" + +# problems: +# +# * if you use a msvc compiled python version (1.5.2) +# 1. you have to insert a __GNUC__ section in its config.h +# 2. you have to generate an import library for its dll +# - create a def-file for python??.dll +# - create an import library using +# dlltool --dllname python15.dll --def python15.def \ +# --output-lib libpython15.a +# +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# +# * We put export_symbols in a def-file, and don't use +# --export-all-symbols because it doesn't worked reliable in some +# tested configurations. And because other windows compilers also +# need their symbols specified this no serious problem. +# +# tested configurations: +# +# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works +# (after patching python's config.h and for C++ some other include files) +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works +# (ld doesn't support -shared, so we use dllwrap) +# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now +# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 +# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html +# - using gcc -mdll instead dllwrap doesn't work without -static because +# it tries to link against dlls instead their import libraries. (If +# it finds the dll first.) +# By specifying -static we force ld to link against the import libraries, +# this is windows standard and there are normally not the necessary symbols +# in the dlls. +# *** only the version of June 2000 shows these problems +# * cygwin gcc 3.2/ld 2.13.90 works +# (ld supports -shared) +# * mingw gcc 3.2/ld 2.13 works +# (ld supports -shared) + +import os +import sys +from subprocess import Popen, PIPE, check_output +import re + +from distutils.unixccompiler import UnixCCompiler +from distutils.errors import CCompilerError +from distutils.version import LooseVersion +from distutils.spawn import find_executable + +def get_msvcr(): + """Include the appropriate MSVC runtime library if Python was built + with MSVC 7.0 or later. + """ + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = sys.version[msc_pos+6:msc_pos+10] + if msc_ver == '1300': + # MSVC 7.0 + return ['msvcr70'] + elif msc_ver == '1310': + # MSVC 7.1 + return ['msvcr71'] + elif msc_ver == '1400': + # VS2005 / MSVC 8.0 + return ['msvcr80'] + elif msc_ver == '1500': + # VS2008 / MSVC 9.0 + return ['msvcr90'] + elif msc_ver == '1600': + # VS2010 / MSVC 10.0 + return ['msvcr100'] + else: + raise ValueError("Unknown MS Compiler version %s " % msc_ver) + + +class CygwinCCompiler(UnixCCompiler): + """ Handles the Cygwin port of the GNU C compiler to Windows. + """ + compiler_type = 'cygwin' + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".dll" + static_lib_format = "lib%s%s" + shared_lib_format = "%s%s" + exe_extension = ".exe" + + def __init__(self, verbose=0, dry_run=0, force=0): + + UnixCCompiler.__init__(self, verbose, dry_run, force) + + status, details = check_config_h() + self.debug_print("Python's GCC status: %s (details: %s)" % + (status, details)) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + "Reason: %s. " + "Compiling may fail because of undefined preprocessor macros." + % details) + + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % + (self.gcc_version, + self.ld_version, + self.dllwrap_version) ) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" + else: + self.linker_dll = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] + self.warn( + "Consider upgrading to a newer version of gcc") + else: + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + + +# the same as cygwin plus some additional parameters +class Mingw32CCompiler(CygwinCCompiler): + """ Handles the Mingw32 port of the GNU C compiler to Windows. + """ + compiler_type = 'mingw32' + + def __init__(self, verbose=0, dry_run=0, force=0): + + CygwinCCompiler.__init__ (self, verbose, dry_run, force) + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. + if self.gcc_version <= "2.91.57": + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + + if is_cygwingcc(): + raise CCompilerError( + 'Cygwin gcc cannot be used with --compiler=mingw32') + + self.set_executables(compiler='gcc -O -Wall', + compiler_so='gcc -mdll -O -Wall', + compiler_cxx='g++ -O -Wall', + linker_exe='gcc', + linker_so='%s %s %s' + % (self.linker_dll, shared_option, + entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) + # (-mthreads: Support thread-safe exception handling on `Mingw32') + + # no additional libraries needed + self.dll_libraries=[] + + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using an unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + """Check if the current Python installation appears amenable to building + extensions with GCC. + + Returns a tuple (status, details), where 'status' is one of the following + constants: + + - CONFIG_H_OK: all is well, go ahead and compile + - CONFIG_H_NOTOK: doesn't look good + - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h + + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + + import sysconfig + + # if sys.version contains GCC then python was compiled with GCC, and the + # pyconfig.h file should be OK + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: + config_h = open(fn) + try: + if "__GNUC__" in config_h.read(): + return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn + else: + return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn + finally: + config_h.close() + except OSError as exc: + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) + +RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') + +def _find_exe_version(cmd): + """Find the version of an executable by running `cmd` in the shell. + + If the command is not found, or the output does not match + `RE_VERSION`, returns None. + """ + executable = cmd.split()[0] + if find_executable(executable) is None: + return None + out = Popen(cmd, shell=True, stdout=PIPE).stdout + try: + out_string = out.read() + finally: + out.close() + result = RE_VERSION.search(out_string) + if result is None: + return None + # LooseVersion works with strings + # so we need to decode our bytes + return LooseVersion(result.group(1).decode()) + +def get_versions(): + """ Try to find out the versions of gcc, ld and dllwrap. + + If not possible it returns None for it. + """ + commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] + return tuple([_find_exe_version(cmd) for cmd in commands]) + +def is_cygwingcc(): + '''Try to determine if the gcc that would be used is from cygwin.''' + out_string = check_output(['gcc', '-dumpmachine']) + return out_string.strip().endswith(b'cygwin') diff --git a/Tools/c-analyzer/distutils/debug.py b/Tools/c-analyzer/distutils/debug.py new file mode 100644 index 000000000000000..daf1660f0d82114 --- /dev/null +++ b/Tools/c-analyzer/distutils/debug.py @@ -0,0 +1,5 @@ +import os + +# If DISTUTILS_DEBUG is anything other than the empty string, we run in +# debug mode. +DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Tools/c-analyzer/distutils/dep_util.py b/Tools/c-analyzer/distutils/dep_util.py new file mode 100644 index 000000000000000..318c830f2eab3e3 --- /dev/null +++ b/Tools/c-analyzer/distutils/dep_util.py @@ -0,0 +1,29 @@ +"""distutils.dep_util + +Utility functions for simple, timestamp-based dependency of files +and groups of files; also, function based entirely on such +timestamp dependency analysis.""" + +import os +from distutils.errors import DistutilsFileError + + +def newer (source, target): + """Return true if 'source' exists and is more recently modified than + 'target', or if 'source' exists and 'target' doesn't. Return false if + both exist and 'target' is the same age or younger than 'source'. + Raise DistutilsFileError if 'source' does not exist. + """ + if not os.path.exists(source): + raise DistutilsFileError("file '%s' does not exist" % + os.path.abspath(source)) + if not os.path.exists(target): + return 1 + + from stat import ST_MTIME + mtime1 = os.stat(source)[ST_MTIME] + mtime2 = os.stat(target)[ST_MTIME] + + return mtime1 > mtime2 + +# newer () diff --git a/Tools/c-analyzer/distutils/errors.py b/Tools/c-analyzer/distutils/errors.py new file mode 100644 index 000000000000000..10f8e316f675f08 --- /dev/null +++ b/Tools/c-analyzer/distutils/errors.py @@ -0,0 +1,48 @@ +"""distutils.errors + +Provides exceptions used by the Distutils modules. Note that Distutils +modules may raise standard exceptions; in particular, SystemExit is +usually raised for errors that are obviously the end-user's fault +(eg. bad command-line arguments). + +This module is safe to use in "from ... import *" mode; it only exports +symbols whose names start with "Distutils" and end with "Error".""" + +class DistutilsError (Exception): + """The root of all Distutils evil.""" + pass + +class DistutilsModuleError (DistutilsError): + """Unable to load an expected module, or to find an expected class + within some module (in particular, command modules and classes).""" + pass + +class DistutilsFileError (DistutilsError): + """Any problems in the filesystem: expected file not found, etc. + Typically this is for problems that we detect before OSError + could be raised.""" + pass + +class DistutilsPlatformError (DistutilsError): + """We don't know how to do something on the current platform (but + we do know how to do it on some platform) -- eg. trying to compile + C files on a platform not supported by a CCompiler subclass.""" + pass + +class DistutilsExecError (DistutilsError): + """Any problems executing an external program (such as the C + compiler, when compiling C files).""" + pass + +# Exception classes used by the CCompiler implementation classes +class CCompilerError (Exception): + """Some compile/link operation failed.""" + +class PreprocessError (CCompilerError): + """Failure to preprocess one or more C/C++ files.""" + +class CompileError (CCompilerError): + """Failure to compile one or more C/C++ source files.""" + +class UnknownFileError (CCompilerError): + """Attempt to process an unknown file type.""" diff --git a/Tools/c-analyzer/distutils/log.py b/Tools/c-analyzer/distutils/log.py new file mode 100644 index 000000000000000..26ecf22ae19befc --- /dev/null +++ b/Tools/c-analyzer/distutils/log.py @@ -0,0 +1,63 @@ +"""A simple log mechanism styled after PEP 282.""" + +# The class here is styled after PEP 282 so that it could later be +# replaced with a standard Python logging implementation. + +DEBUG = 1 +INFO = 2 +WARN = 3 +ERROR = 4 +FATAL = 5 + +import sys + +class Log: + + def __init__(self, threshold=WARN): + self.threshold = threshold + + def _log(self, level, msg, args): + if level not in (DEBUG, INFO, WARN, ERROR, FATAL): + raise ValueError('%s wrong log level' % str(level)) + + if level >= self.threshold: + if args: + msg = msg % args + if level in (WARN, ERROR, FATAL): + stream = sys.stderr + else: + stream = sys.stdout + try: + stream.write('%s\n' % msg) + except UnicodeEncodeError: + # emulate backslashreplace error handler + encoding = stream.encoding + msg = msg.encode(encoding, "backslashreplace").decode(encoding) + stream.write('%s\n' % msg) + stream.flush() + + def log(self, level, msg, *args): + self._log(level, msg, args) + + def debug(self, msg, *args): + self._log(DEBUG, msg, args) + + def info(self, msg, *args): + self._log(INFO, msg, args) + + def warn(self, msg, *args): + self._log(WARN, msg, args) + + def error(self, msg, *args): + self._log(ERROR, msg, args) + + def fatal(self, msg, *args): + self._log(FATAL, msg, args) + +_global_log = Log() +log = _global_log.log +debug = _global_log.debug +info = _global_log.info +warn = _global_log.warn +error = _global_log.error +fatal = _global_log.fatal diff --git a/Tools/c-analyzer/distutils/msvc9compiler.py b/Tools/c-analyzer/distutils/msvc9compiler.py new file mode 100644 index 000000000000000..38fff9b2d531209 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvc9compiler.py @@ -0,0 +1,438 @@ +"""distutils.msvc9compiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes + +import os +import subprocess +import sys +import re + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +import winreg + +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error + +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + +NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) +if NATIVE_WIN64: + # Visual C++ is a 32-bit application, so we need to look in + # the corresponding registry branch, if we're running a + # 64-bit Python on Win64 + VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" +else: + VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targeting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', +} + +class Reg: + """Helper class to read values from the registry + """ + + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) + + def read_values(cls, base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.vsbase = VS_BASE % version + self.load_macros(version) + + def set_macro(self, macro, path, key): + self.macros["$(%s)" % macro] = Reg.get_value(path, key) + + def load_macros(self, version): + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") + try: + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") + else: + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2008 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + log.debug("Unable to find productdir in registry") + productdir = None + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = {"include", "lib", "libpath", "path"} + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + finally: + popen.stdout.close() + popen.stderr.close() + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name + self.initialized = False + + # -- Worker methods ------------------------------------------------ + + def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): + # If we need a manifest at all, an embedded manifest is recommended. + # See MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can check it, and possibly embed it, later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + + def manifest_get_embed_info(self, target_desc, ld_args): + # If a manifest should be embedded, return a tuple of + # (manifest_filename, resource_id). Returns None if no manifest + # should be embedded. See http://bugs.python.org/issue7833 for why + # we want to avoid any manifest for extension modules if we can. + for arg in ld_args: + if arg.startswith("/MANIFESTFILE:"): + temp_manifest = arg.split(":", 1)[1] + break + else: + # no /MANIFESTFILE so nothing to do. + return None + if target_desc == CCompiler.EXECUTABLE: + # by default, executables always get the manifest with the + # CRT referenced. + mfid = 1 + else: + # Extension modules try and avoid any manifest if possible. + mfid = 2 + temp_manifest = self._remove_visual_c_ref(temp_manifest) + if temp_manifest is None: + return None + return temp_manifest, mfid + + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + # Returns either the filename of the modified manifest or + # None if no manifest should be embedded. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = r"\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + # Now see if any other assemblies are referenced - if not, we + # don't want a manifest embedded. + pattern = re.compile( + r"""|)""", re.DOTALL) + if re.search(pattern, manifest_buf) is None: + return None + + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + return manifest_file + finally: + manifest_f.close() + except OSError: + pass + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe diff --git a/Tools/c-analyzer/distutils/msvccompiler.py b/Tools/c-analyzer/distutils/msvccompiler.py new file mode 100644 index 000000000000000..c0864b1f95d4e22 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvccompiler.py @@ -0,0 +1,327 @@ +"""distutils.msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) + +import sys, os +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +_can_read_reg = False +try: + import winreg + + _can_read_reg = True + hkey_mod = winreg + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error + +except ImportError: + try: + import win32api + import win32con + _can_read_reg = True + hkey_mod = win32con + + RegOpenKeyEx = win32api.RegOpenKeyEx + RegEnumKey = win32api.RegEnumKey + RegEnumValue = win32api.RegEnumValue + RegError = win32api.error + except ImportError: + log.info("Warning: Can't read registry to find the " + "necessary compiler setting\n" + "Make sure that Python modules winreg, " + "win32api or win32con are installed.") + +if _can_read_reg: + HKEYS = (hkey_mod.HKEY_USERS, + hkey_mod.HKEY_CURRENT_USER, + hkey_mod.HKEY_LOCAL_MACHINE, + hkey_mod.HKEY_CLASSES_ROOT) + +def read_keys(base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + +def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i += 1 + return d + +def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + +class MacroExpander: + def __init__(self, version): + self.macros = {} + self.load_macros(version) + + def set_macro(self, macro, path, key): + for base in HKEYS: + d = read_values(base, path) + if d: + self.macros["$(%s)" % macro] = d[key] + break + + def load_macros(self, version): + vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version + self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") + net = r"Software\Microsoft\.NETFramework" + self.set_macro("FrameworkDir", net, "installroot") + try: + if version > 7.0: + self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + else: + self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") + except KeyError as exc: # + raise DistutilsPlatformError( + """Python was built with Visual Studio 2003; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2003 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = read_values(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def get_build_architecture(): + """Return the processor architecture. + + Possible results are "Intel" or "AMD64". + """ + + prefix = " bit (" + i = sys.version.find(prefix) + if i == -1: + return "Intel" + j = sys.version.find(")", i) + return sys.version[i+len(prefix):j] + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = get_build_version() + self.__arch = get_build_architecture() + if self.__arch == "Intel": + # x86 + if self.__version >= 7: + self.__root = r"Software\Microsoft\VisualStudio" + self.__macros = MacroExpander(self.__version) + else: + self.__root = r"Software\Microsoft\Devstudio" + self.__product = "Visual Studio version %s" % self.__version + else: + # Win64. Assume this was built with the platform SDK + self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) + + self.initialized = False + + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe + + def get_msvc_paths(self, path, platform='x86'): + """Get a list of devstudio directories (include, lib or path). + + Return a list of strings. The list will be empty if unable to + access the registry or appropriate registry keys not found. + """ + if not _can_read_reg: + return [] + + path = path + " dirs" + if self.__version >= 7: + key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" + % (self.__root, self.__version)) + else: + key = (r"%s\6.0\Build System\Components\Platforms" + r"\Win32 (%s)\Directories" % (self.__root, platform)) + + for base in HKEYS: + d = read_values(base, key) + if d: + if self.__version >= 7: + return self.__macros.sub(d[path]).split(";") + else: + return d[path].split(";") + # MSVC 6 seems to create the registry entries we need only when + # the GUI is run. + if self.__version == 6: + for base in HKEYS: + if read_values(base, r"%s\6.0" % self.__root) is not None: + self.warn("It seems you have Visual Studio 6 installed, " + "but the expected registry settings are not present.\n" + "You must at least run the Visual Studio GUI once " + "so that these entries are created.") + break + return [] + + def set_path_env_var(self, name): + """Set environment variable 'name' to an MSVC path type value. + + This is equivalent to a SET command prior to execution of spawned + commands. + """ + + if name == "lib": + p = self.get_msvc_paths("library") + else: + p = self.get_msvc_paths(name) + if p: + os.environ[name] = ';'.join(p) + + +if get_build_version() >= 8.0: + log.debug("Importing new compiler from distutils.msvc9compiler") + OldMSVCCompiler = MSVCCompiler + from distutils.msvc9compiler import MSVCCompiler + # get_build_architecture not really relevant now we support cross-compile + from distutils.msvc9compiler import MacroExpander diff --git a/Tools/c-analyzer/distutils/spawn.py b/Tools/c-analyzer/distutils/spawn.py new file mode 100644 index 000000000000000..eb9794964d7acb9 --- /dev/null +++ b/Tools/c-analyzer/distutils/spawn.py @@ -0,0 +1,48 @@ +"""distutils.spawn + +Provides the 'spawn()' function, a front-end to various platform- +specific functions for launching another program in a sub-process. +Also provides the 'find_executable()' to search the path for a given +executable name. +""" + +import sys +import os +import os.path + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None diff --git a/Tools/c-analyzer/distutils/unixccompiler.py b/Tools/c-analyzer/distutils/unixccompiler.py new file mode 100644 index 000000000000000..1a3a8e5255070d2 --- /dev/null +++ b/Tools/c-analyzer/distutils/unixccompiler.py @@ -0,0 +1,102 @@ +"""distutils.unixccompiler + +Contains the UnixCCompiler class, a subclass of CCompiler that handles +the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys, re + +from distutils.dep_util import newer +from distutils.ccompiler import CCompiler, gen_preprocess_options +from distutils.errors import DistutilsExecError, CompileError + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the setup.py command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + + +class UnixCCompiler(CCompiler): + + compiler_type = 'unix' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["cc"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + fixed_args = self._fix_compile_args(None, macros, include_dirs) + ignore, macros, include_dirs = fixed_args + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(['-o', output_file]) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + raise CompileError(msg) diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py new file mode 100644 index 000000000000000..89ca094336fdb8f --- /dev/null +++ b/Tools/c-analyzer/distutils/util.py @@ -0,0 +1,171 @@ +"""distutils.util + +Miscellaneous utility functions -- anything that doesn't fit into +one of the other *util.py modules. +""" + +import os +import re +import string +import sys +from distutils.errors import DistutilsPlatformError + +def get_host_platform(): + """Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; eg. on Linux, the kernel version isn't + particularly important. + + Examples of returned values: + linux-i586 + linux-alpha (?) + solaris-2.6-sun4u + + Windows will return one of: + win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win32 (all others - specifically, sys.platform is returned) + + For other non-POSIX platforms, currently just returns 'sys.platform'. + + """ + if os.name == 'nt': + if 'amd64' in sys.version.lower(): + return 'win-amd64' + if '(arm)' in sys.version.lower(): + return 'win-arm32' + if '(arm64)' in sys.version.lower(): + return 'win-arm64' + return sys.platform + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + return os.environ["_PYTHON_HOST_PLATFORM"] + + if os.name != "posix" or not hasattr(os, 'uname'): + # XXX what about the architecture? NT is Intel or Alpha, + # Mac OS is M68k or PPC, etc. + return sys.platform + + # Try to distinguish various flavours of Unix + + (osname, host, release, version, machine) = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname[:5] == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return "%s-%s" % (osname, machine) + elif osname[:5] == "sunos": + if release[0] >= "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = "%d.%s" % (int(release[0]) - 3, release[2:]) + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += ".%s" % bitness[sys.maxsize] + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + rel_re = re.compile (r'[\d.]+', re.ASCII) + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + import _osx_support, sysconfig + osname, release, machine = _osx_support.get_platform_osx( + sysconfig.get_config_vars(), + osname, release, machine) + + return "%s-%s-%s" % (osname, release, machine) + +def get_platform(): + if os.name == 'nt': + TARGET_TO_PLAT = { + 'x86' : 'win32', + 'x64' : 'win-amd64', + 'arm' : 'win-arm32', + } + return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform() + else: + return get_host_platform() + + +# Needed by 'split_quoted()' +_wordchars_re = _squote_re = _dquote_re = None +def _init_regex(): + global _wordchars_re, _squote_re, _dquote_re + _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) + _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") + _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') + +def split_quoted (s): + """Split a string up according to Unix shell-like rules for quotes and + backslashes. In short: words are delimited by spaces, as long as those + spaces are not escaped by a backslash, or inside a quoted string. + Single and double quotes are equivalent, and the quote characters can + be backslash-escaped. The backslash is stripped from any two-character + escape sequence, leaving only the escaped character. The quote + characters are stripped from any quoted string. Returns a list of + words. + """ + + # This is a nice algorithm for splitting up a single string, since it + # doesn't require character-by-character examination. It was a little + # bit of a brain-bender to get it working right, though... + if _wordchars_re is None: _init_regex() + + s = s.strip() + words = [] + pos = 0 + + while s: + m = _wordchars_re.match(s, pos) + end = m.end() + if end == len(s): + words.append(s[:end]) + break + + if s[end] in string.whitespace: # unescaped, unquoted whitespace: now + words.append(s[:end]) # we definitely have a word delimiter + s = s[end:].lstrip() + pos = 0 + + elif s[end] == '\\': # preserve whatever is being escaped; + # will become part of the current word + s = s[:end] + s[end+1:] + pos = end+1 + + else: + if s[end] == "'": # slurp singly-quoted string + m = _squote_re.match(s, end) + elif s[end] == '"': # slurp doubly-quoted string + m = _dquote_re.match(s, end) + else: + raise RuntimeError("this can't happen (bad char '%c')" % s[end]) + + if m is None: + raise ValueError("bad string (mismatched %s quotes?)" % s[end]) + + (beg, end) = m.span() + s = s[:beg] + s[beg+1:end-1] + s[end:] + pos = m.end() - 2 + + if pos >= len(s): + words.append(s) + break + + return words + +# split_quoted () From c6858d1e7f4cd3184d5ddea4025ad5dfc7596546 Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Thu, 9 Mar 2023 22:09:12 +0100 Subject: [PATCH 20/35] gh-102255: Improve build support for Windows API partitions (GH-102256) Add `MS_WINDOWS_DESKTOP`, `MS_WINDOWS_APPS`, `MS_WINDOWS_SYSTEM` and `MS_WINDOWS_GAMES` preprocessor definitions to allow switching off functionality missing from particular API partitions ("partitions" are used in Windows to identify overlapping subsets of APIs). CPython only officially supports `MS_WINDOWS_DESKTOP` and `MS_WINDOWS_SYSTEM` (APPS is included by normal desktop builds, but APPS without DESKTOP is not covered). Other configurations are a convenience for people building their own runtimes. `MS_WINDOWS_GAMES` is for the Xbox subset of the Windows API, which is also available on client OS, but is restricted compared to `MS_WINDOWS_DESKTOP`. These restrictions may change over time, as they relate to the build headers rather than the OS support, and so we assume that Xbox builds will use the latest available version of the GDK. --- Include/internal/pycore_fileutils.h | 8 + Lib/test/test_os.py | 8 +- ...-02-26-11-43-56.gh-issue-102255.cRnI5x.rst | 1 + Modules/_io/_iomodule.c | 6 +- Modules/_io/_iomodule.h | 6 +- Modules/_io/clinic/winconsoleio.c.h | 42 ++-- Modules/_io/fileio.c | 2 + Modules/_io/winconsoleio.c | 6 +- Modules/_localemodule.c | 8 +- Modules/_multiprocessing/multiprocessing.h | 4 +- Modules/_pickle.c | 6 + Modules/_randommodule.c | 6 +- Modules/_ssl.c | 4 + Modules/_winapi.c | 13 ++ Modules/clinic/posixmodule.c.h | 10 +- Modules/errnomodule.c | 2 + Modules/faulthandler.c | 2 +- Modules/getpath.c | 2 + Modules/mmapmodule.c | 8 +- Modules/posixmodule.c | 58 +++-- Modules/selectmodule.c | 6 +- Modules/socketmodule.c | 60 +++-- Modules/timemodule.c | 8 +- Objects/obmalloc.c | 1 - PC/clinic/msvcrtmodule.c.h | 42 +++- PC/clinic/winreg.c.h | 218 +++++++++++++++++- PC/config.c | 8 + PC/config_minimal.c | 6 + PC/msvcrtmodule.c | 22 ++ PC/pyconfig.h | 24 +- PC/winreg.c | 25 ++ Parser/myreadline.c | 8 +- Python/dynload_win.c | 8 + Python/fileutils.c | 89 ++++++- Python/pylifecycle.c | 2 +- Python/sysmodule.c | 4 + 36 files changed, 633 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index f8e2bf225908887..445ac0a3d955d93 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -251,6 +251,14 @@ extern int _Py_add_relfile(wchar_t *dirname, extern size_t _Py_find_basename(const wchar_t *filename); PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); +// The Windows Games API family does not provide these functions +// so provide our own implementations. Remove them in case they get added +// to the Games API family +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +#include + +extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 792794ca1094892..ba6feb69ea17125 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3084,11 +3084,13 @@ def test_device_encoding(self): class PidTests(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") def test_getppid(self): - p = subprocess.Popen([sys.executable, '-c', + p = subprocess.Popen([sys._base_executable, '-c', 'import os; print(os.getppid())'], - stdout=subprocess.PIPE) - stdout, _ = p.communicate() + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, error = p.communicate() # We are the parent of our subprocess + self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) def check_waitpid(self, code, exitcode, callback=None): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst new file mode 100644 index 000000000000000..daabc3c15f6ee2a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst @@ -0,0 +1 @@ +Improve build support for the Xbox. Patch by Max Bachmann. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index d8d836b8382eb18..1506755427fc0d6 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -317,7 +317,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, _PyIO_State *state = get_io_state(module); { PyObject *RawIO_class = (PyObject *)state->PyFileIO_Type; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; @@ -660,7 +660,7 @@ static PyTypeObject* static_types[] = { // PyRawIOBase_Type(PyIOBase_Type) subclasses &_PyBytesIOBuffer_Type, -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO &PyWindowsConsoleIO_Type, #endif }; @@ -718,7 +718,7 @@ PyInit__io(void) } // Set type base classes -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; #endif diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 02daef9e85677e8..d7224e56f9a7224 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -26,9 +26,9 @@ extern PyType_Spec fileio_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -178,7 +178,7 @@ find_io_state_by_def(PyTypeObject *type) extern _PyIO_State *_PyIO_get_module_state(void); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index df834dbde40f5b2..4c5cd0892c4a6d2 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -8,7 +8,7 @@ preserve #endif -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__, "close($self, /)\n" @@ -31,9 +31,9 @@ _io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_close_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n" @@ -131,9 +131,9 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__, "fileno($self, /)\n" @@ -153,9 +153,9 @@ _io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_fileno_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__, "readable($self, /)\n" @@ -175,9 +175,9 @@ _io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__, "writable($self, /)\n" @@ -197,9 +197,9 @@ _io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_writable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__, "readinto($self, buffer, /)\n" @@ -239,9 +239,9 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__, "readall($self, /)\n" @@ -263,9 +263,9 @@ _io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readall_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__, "read($self, size=-1, /)\n" @@ -305,9 +305,9 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyObject *const *args, Py_ssize_t return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__, "write($self, b, /)\n" @@ -348,9 +348,9 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__, "isatty($self, /)\n" @@ -370,7 +370,7 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_isatty_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ #ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF @@ -407,4 +407,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=4920e9068e0cf08a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=163e934aa9b0ef16 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 35a498ce5a83544..1118d86e6c9a10f 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -37,7 +37,9 @@ #ifdef MS_WINDOWS /* can simulate truncate with Win32 API functions; see file_truncate */ #define HAVE_FTRUNCATE +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index de07b50f5ce4cbd..f836e230243020c 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -11,7 +11,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO #include "structmember.h" // PyMemberDef #ifdef HAVE_SYS_TYPES_H @@ -22,7 +22,9 @@ #endif #include /* For offsetof */ +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #include @@ -1177,4 +1179,4 @@ PyTypeObject PyWindowsConsoleIO_Type = { 0, /* tp_finalize */ }; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 23c38e14d997d1b..96675cdfb661ad7 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -35,7 +35,9 @@ This software comes with no warranty. Use at your own risk. #endif #if defined(MS_WINDOWS) +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif @@ -457,12 +459,12 @@ _locale__getdefaultlocale_impl(PyObject *module) PyOS_snprintf(encoding, sizeof(encoding), "cp%u", GetACP()); - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO639LANGNAME, locale, sizeof(locale))) { Py_ssize_t i = strlen(locale); locale[i++] = '_'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO3166CTRYNAME, locale+i, (int)(sizeof(locale)-i))) return Py_BuildValue("ss", locale, encoding); @@ -474,7 +476,7 @@ _locale__getdefaultlocale_impl(PyObject *module) locale[0] = '0'; locale[1] = 'x'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, locale+2, sizeof(locale)-2)) { return Py_BuildValue("ss", locale, encoding); } diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index b595e5a8dd18de4..dfc2a8e0799a603 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -12,7 +12,9 @@ */ #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # include # include /* getpid() */ diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 1b34977806b661c..a26732af8ba2a1f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -42,6 +42,12 @@ enum { #define FLOAT FLOAT_ #define INT INT_ #define LONG LONG_ + +/* This can already be defined on Windows to set the character set + the Windows header files treat as default */ +#ifdef UNICODE +#undef UNICODE +#endif #endif /* Pickle opcodes. These must be kept updated with pickle.py. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 68060c07033d345..6e22053239305a5 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -77,6 +77,10 @@ # include // getpid() #endif +#ifdef MS_WINDOWS +# include +#endif + /* Period parameters -- These are all magic. Don't change. */ #define N 624 #define M 397 @@ -259,7 +263,7 @@ random_seed_time_pid(RandomObject *self) key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); -#ifdef MS_WINDOWS_NON_DESKTOP +#if defined(MS_WINDOWS) && !defined(MS_WINDOWS_DESKTOP) && !defined(MS_WINDOWS_SYSTEM) key[2] = (uint32_t)GetCurrentProcessId(); #elif defined(HAVE_GETPID) key[2] = (uint32_t)getpid(); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8f03a846aed0891..28112317bc289ea 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,6 +28,10 @@ /* Include symbols from _socket module */ #include "socketmodule.h" +#ifdef MS_WINDOWS +# include +#endif + #include "_ssl.h" /* Redefined below for Windows debug builds after important #includes */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eefc2571ee8287d..83cde7501176b64 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -39,8 +39,11 @@ #include "structmember.h" // PyMemberDef +#ifndef WINDOWS_LEAN_AND_MEAN #define WINDOWS_LEAN_AND_MEAN +#endif #include "windows.h" +#include #include #include "winreparse.h" @@ -63,6 +66,14 @@ #define T_HANDLE T_POINTER +// winbase.h limits the STARTF_* flags to the desktop API as of 10.0.19041. +#ifndef STARTF_USESHOWWINDOW +#define STARTF_USESHOWWINDOW 0x00000001 +#endif +#ifndef STARTF_USESTDHANDLES +#define STARTF_USESTDHANDLES 0x00000100 +#endif + typedef struct { PyTypeObject *overlapped_type; } WinApiState; @@ -1201,8 +1212,10 @@ _winapi_ExitProcess_impl(PyObject *module, UINT ExitCode) /*[clinic end generated code: output=a387deb651175301 input=4f05466a9406c558]*/ { #if defined(Py_DEBUG) +#ifdef MS_WINDOWS_DESKTOP SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOALIGNMENTFAULTEXCEPT| SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX); +#endif _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); #endif diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dcd25c28370c930..6565f8df935cb85 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10988,7 +10988,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #endif /* defined(HAVE_GETRANDOM_SYSCALL) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__add_dll_directory__doc__, "_add_dll_directory($module, /, path)\n" @@ -11057,9 +11057,9 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__remove_dll_directory__doc__, "_remove_dll_directory($module, /, cookie)\n" @@ -11120,7 +11120,7 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ #if (defined(WIFEXITED) || defined(MS_WINDOWS)) @@ -11796,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 4de4144520aa48e..df4e494ba8a9737 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -5,7 +5,9 @@ /* Windows socket errors (WSA*) */ #ifdef MS_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include /* The following constants were added to errno.h in VS2010 but have preferred WSA equivalents. */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 5309a3728c5e07b..bfe35fed7a450a3 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -953,7 +953,7 @@ faulthandler_unregister_py(PyObject *self, PyObject *args) static void faulthandler_suppress_crash_report(void) { -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP UINT mode; /* Configure Windows to not display the Windows Error Reporting dialog */ diff --git a/Modules/getpath.c b/Modules/getpath.c index c807a3c8e9a2b90..2f20521592ce2e2 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -745,10 +745,12 @@ static int library_to_dict(PyObject *dict, const char *key) { #ifdef MS_WINDOWS +#ifdef Py_ENABLE_SHARED extern HMODULE PyWin_DLLhModule; if (PyWin_DLLhModule) { return winmodule_to_dict(dict, key, PyWin_DLLhModule); } +#endif #elif defined(WITH_NEXT_FRAMEWORK) static char modPath[MAXPATHLEN + 1]; static int modPathInitialized = -1; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6054c2853d7a78d..fe76ca6eafaa88a 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -29,6 +29,10 @@ #include "structmember.h" // PyMemberDef #include // offsetof() +// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA +// need to be replaced with OpenFileMappingW / CreateFileMappingW +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) + #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H @@ -647,7 +651,7 @@ mmap_flush_method(mmap_object *self, PyObject *args) if (self->access == ACCESS_READ || self->access == ACCESS_COPY) Py_RETURN_NONE; -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) if (!FlushViewOfFile(self->data+offset, size)) { PyErr_SetFromWindowsErr(GetLastError()); return NULL; @@ -1724,3 +1728,5 @@ PyInit_mmap(void) { return PyModuleDef_Init(&mmapmodule); } + +#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a3d86cbe7a57fb2..0d534f35f6a420a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -26,11 +26,15 @@ #ifdef MS_WINDOWS # include -# include +# if !defined(MS_WINDOWS_GAMES) || defined(MS_WINDOWS_DESKTOP) +# include +# endif # include # include // UNLEN # include "osdefs.h" // SEP -# define HAVE_SYMLINK +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +# define HAVE_SYMLINK +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ #endif #include "structmember.h" // PyMemberDef @@ -311,7 +315,7 @@ corresponding Unix manual entries for more information on calls."); # include #endif -#if defined(MS_WINDOWS) +#ifdef HAVE_WINDOWS_CONSOLE_IO # define TERMSIZE_USE_CONIO #elif defined(HAVE_SYS_IOCTL_H) # include @@ -321,7 +325,7 @@ corresponding Unix manual entries for more information on calls."); # if defined(TIOCGWINSZ) # define TERMSIZE_USE_IOCTL # endif -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ @@ -329,21 +333,25 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_OPENDIR 1 # define HAVE_SYSTEM 1 # include -#else -# ifdef _MSC_VER - /* Microsoft compiler */ +#elif defined( _MSC_VER) + /* Microsoft compiler */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_GETPPID 1 +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_APP | MS_WINDOWS_SYSTEM */ +# if defined(MS_WINDOWS_DESKTOP) # define HAVE_GETLOGIN 1 +# endif /* MS_WINDOWS_DESKTOP */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SPAWNV 1 # define HAVE_EXECV 1 # define HAVE_WSPAWNV 1 # define HAVE_WEXECV 1 -# define HAVE_PIPE 1 # define HAVE_SYSTEM 1 # define HAVE_CWAIT 1 -# define HAVE_FSYNC 1 -# define fsync _commit -# endif /* _MSC_VER */ +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ +# define HAVE_PIPE 1 +# define HAVE_FSYNC 1 +# define fsync _commit #endif /* ! __WATCOMC__ || __QNX__ */ /*[clinic input] @@ -1536,7 +1544,7 @@ convertenviron(void) #ifdef MS_WINDOWS /* _wenviron must be initialized in this way if the program is started through main() instead of wmain(). */ - _wgetenv(L""); + (void)_wgetenv(L""); e = _wenviron; #elif defined(WITH_NEXT_FRAMEWORK) || (defined(__APPLE__) && defined(Py_ENABLE_SHARED)) /* environ is not accessible as an extern in a shared object on OSX; use @@ -1785,6 +1793,10 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re if (n && (pszFile[n - 1] == L'\\' || pszFile[n - 1] == L'/')) { // cannot use PyMem_Malloc here because we do not hold the GIL filename = (LPCWSTR)malloc((n + 1) * sizeof(filename[0])); + if(!filename) { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return FALSE; + } wcsncpy_s((LPWSTR)filename, n + 1, pszFile, n); while (--n > 0 && (filename[n] == L'\\' || filename[n] == L'/')) { ((LPWSTR)filename)[n] = L'\0'; @@ -7933,10 +7945,10 @@ static PyObject * os_getpid_impl(PyObject *module) /*[clinic end generated code: output=9ea6fdac01ed2b3c input=5a9a00f0ab68aa00]*/ { -#ifdef MS_WINDOWS_NON_DESKTOP - return PyLong_FromUnsignedLong(GetCurrentProcessId()); -#else +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) return PyLong_FromPid(getpid()); +#else + return PyLong_FromUnsignedLong(GetCurrentProcessId()); #endif } #endif /* defined(HAVE_GETPID) */ @@ -8392,6 +8404,7 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) DWORD err; HANDLE handle; +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Console processes which share a common console can be sent CTRL+C or CTRL+BREAK events, provided they handle said events. */ if (sig == CTRL_C_EVENT || sig == CTRL_BREAK_EVENT) { @@ -8399,9 +8412,11 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) err = GetLastError(); PyErr_SetFromWindowsErr(err); } - else + else { Py_RETURN_NONE; + } } +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* If the signal is outside of what GenerateConsoleCtrlEvent can use, attempt to open and terminate the process. */ @@ -13776,7 +13791,9 @@ os_cpu_count_impl(PyObject *module) { int ncpu = 0; #ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); +#endif #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) @@ -13848,6 +13865,10 @@ os_set_inheritable_impl(PyObject *module, int fd, int inheritable) #ifdef MS_WINDOWS +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif + /*[clinic input] os.get_handle_inheritable -> bool handle: intptr_t @@ -15023,7 +15044,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) } #endif /* HAVE_GETRANDOM_SYSCALL */ -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /* bpo-36085: Helper functions for managing DLL search directories * on win32 */ @@ -15114,7 +15136,7 @@ os__remove_dll_directory_impl(PyObject *module, PyObject *cookie) Py_RETURN_NONE; } -#endif +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ /* Only check if WIFEXITED is available: expect that it comes diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index df4043de08daccc..5a1e40d0b4a482c 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -57,8 +57,10 @@ extern void bzero(void *, int); #endif #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN -# include +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include #else # define SOCKET int #endif diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 43a0cc0f963f9d9..b7927750e334b77 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -270,7 +270,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include -#else +#else /* MS_WINDOWS */ /* MS_WINDOWS includes */ # ifdef HAVE_FCNTL_H @@ -281,7 +281,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ -#ifdef MS_WINDOWS #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -312,7 +311,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only -#endif /* MS_WINDOWS */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -348,13 +346,18 @@ remove_unusable_flags(PyObject *m) { PyObject *dict; OSVERSIONINFOEX info; - DWORDLONG dwlConditionMask; dict = PyModule_GetDict(m); if (dict == NULL) { return -1; } - +#ifndef MS_WINDOWS_DESKTOP + info.dwOSVersionInfoSize = sizeof(info); + if (!GetVersionExW((OSVERSIONINFOW*) &info)) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#else /* set to Windows 10, except BuildNumber. */ memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); @@ -362,19 +365,30 @@ remove_unusable_flags(PyObject *m) info.dwMinorVersion = 0; /* set Condition Mask */ - dwlConditionMask = 0; + DWORDLONG dwlConditionMask = 0; VER_SET_CONDITION(dwlConditionMask, VER_MAJORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_MINORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_BUILDNUMBER, VER_GREATER_EQUAL); +#endif for (int i=0; i 10 || + (info.dwMajorVersion == 10 && info.dwMinorVersion > 0) || + (info.dwMajorVersion == 10 && info.dwMinorVersion == 0 && + info.dwBuildNumber >= win_runtime_flags[i].build_number); +#endif + if (isSupported) { break; } else { @@ -497,14 +511,14 @@ remove_unusable_flags(PyObject *m) #endif #endif -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP #define sockaddr_rc SOCKADDR_BTH_REDEF #define USE_BLUETOOTH 1 #define AF_BLUETOOTH AF_BTH #define BTPROTO_RFCOMM BTHPROTO_RFCOMM #define _BT_RC_MEMB(sa, memb) ((sa)->memb) -#endif +#endif /* MS_WINDOWS_DESKTOP */ /* Convert "sock_addr_t *" to "struct sockaddr *". */ #define SAS2SA(x) (&((x)->sa)) @@ -2869,11 +2883,16 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) newfd = ctx.result; #ifdef MS_WINDOWS +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { PyErr_SetFromWindowsErr(0); SOCKETCLOSE(newfd); goto finally; } +#endif #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -5434,11 +5453,6 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, proto = 0; } #ifdef MS_WINDOWS - /* Windows implementation */ -#ifndef WSA_FLAG_NO_HANDLE_INHERIT -#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 -#endif - Py_BEGIN_ALLOW_THREADS fd = WSASocketW(family, type, proto, NULL, 0, @@ -6150,8 +6164,9 @@ socket_dup(PyObject *self, PyObject *fdobj) #endif fd = PyLong_AsSocket_t(fdobj); - if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) + if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) { return NULL; + } #ifdef MS_WINDOWS if (WSADuplicateSocketW(fd, GetCurrentProcessId(), &info)) @@ -6160,8 +6175,9 @@ socket_dup(PyObject *self, PyObject *fdobj) newfd = WSASocketW(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &info, 0, WSA_FLAG_OVERLAPPED); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return set_error(); + } if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { closesocket(newfd); @@ -6171,13 +6187,15 @@ socket_dup(PyObject *self, PyObject *fdobj) #else /* On UNIX, dup can be used to duplicate the file descriptor of a socket */ newfd = _Py_dup(fd); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return NULL; + } #endif newfdobj = PyLong_FromSocket_t(newfd); - if (newfdobj == NULL) + if (newfdobj == NULL) { SOCKETCLOSE(newfd); + } return newfdobj; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c2bacaae0c03395..c50e689bb6986ca 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -30,7 +30,9 @@ # include #else # ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ @@ -1135,7 +1137,9 @@ time_tzset(PyObject *self, PyObject *unused) return NULL; } +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif /* Reset timezone, altzone, daylight and tzname */ if (init_timezone(m) < 0) { @@ -1753,7 +1757,9 @@ init_timezone(PyObject *m) */ #ifdef HAVE_DECL_TZNAME PyObject *otz0, *otz1; +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif PyModule_AddIntConstant(m, "timezone", _Py_timezone); #ifdef HAVE_ALTZONE PyModule_AddIntConstant(m, "altzone", altzone); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 276c5a276c06e60..5e1bcda1d976bb2 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -8,7 +8,6 @@ #include // malloc() #include - #undef uint #define uint pymem_uint diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index d808ef0bbd0ffe4..b708c6cdde757c6 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -261,6 +261,8 @@ msvcrt_getch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwch__doc__, "getwch($module, /)\n" "--\n" @@ -285,6 +287,8 @@ msvcrt_getwch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_getche__doc__, "getche($module, /)\n" "--\n" @@ -309,6 +313,8 @@ msvcrt_getche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwche__doc__, "getwche($module, /)\n" "--\n" @@ -333,6 +339,8 @@ msvcrt_getwche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_putch__doc__, "putch($module, char, /)\n" "--\n" @@ -367,6 +375,8 @@ msvcrt_putch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_putwch__doc__, "putwch($module, unicode_char, /)\n" "--\n" @@ -403,6 +413,8 @@ msvcrt_putwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_ungetch__doc__, "ungetch($module, char, /)\n" "--\n" @@ -441,6 +453,8 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_ungetwch__doc__, "ungetwch($module, unicode_char, /)\n" "--\n" @@ -477,6 +491,8 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + #if defined(_DEBUG) PyDoc_STRVAR(msvcrt_CrtSetReportFile__doc__, @@ -610,6 +626,8 @@ msvcrt_set_error_mode(PyObject *module, PyObject *arg) #endif /* defined(_DEBUG) */ +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(msvcrt_GetErrorMode__doc__, "GetErrorMode($module, /)\n" "--\n" @@ -628,6 +646,8 @@ msvcrt_GetErrorMode(PyObject *module, PyObject *Py_UNUSED(ignored)) return msvcrt_GetErrorMode_impl(module); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ + PyDoc_STRVAR(msvcrt_SetErrorMode__doc__, "SetErrorMode($module, mode, /)\n" "--\n" @@ -656,6 +676,22 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) return return_value; } +#ifndef MSVCRT_GETWCH_METHODDEF + #define MSVCRT_GETWCH_METHODDEF +#endif /* !defined(MSVCRT_GETWCH_METHODDEF) */ + +#ifndef MSVCRT_GETWCHE_METHODDEF + #define MSVCRT_GETWCHE_METHODDEF +#endif /* !defined(MSVCRT_GETWCHE_METHODDEF) */ + +#ifndef MSVCRT_PUTWCH_METHODDEF + #define MSVCRT_PUTWCH_METHODDEF +#endif /* !defined(MSVCRT_PUTWCH_METHODDEF) */ + +#ifndef MSVCRT_UNGETWCH_METHODDEF + #define MSVCRT_UNGETWCH_METHODDEF +#endif /* !defined(MSVCRT_UNGETWCH_METHODDEF) */ + #ifndef MSVCRT_CRTSETREPORTFILE_METHODDEF #define MSVCRT_CRTSETREPORTFILE_METHODDEF #endif /* !defined(MSVCRT_CRTSETREPORTFILE_METHODDEF) */ @@ -667,4 +703,8 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_SET_ERROR_MODE_METHODDEF #define MSVCRT_SET_ERROR_MODE_METHODDEF #endif /* !defined(MSVCRT_SET_ERROR_MODE_METHODDEF) */ -/*[clinic end generated code: output=204bae9fee7f6124 input=a9049054013a1b77]*/ + +#ifndef MSVCRT_GETERRORMODE_METHODDEF + #define MSVCRT_GETERRORMODE_METHODDEF +#endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ +/*[clinic end generated code: output=2db6197608a6aab3 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 2834d9967a77260..7a9474301da8a1f 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -8,6 +8,8 @@ preserve #endif +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Close__doc__, "Close($self, /)\n" "--\n" @@ -28,6 +30,10 @@ winreg_HKEYType_Close(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Close_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Detach__doc__, "Detach($self, /)\n" "--\n" @@ -54,6 +60,10 @@ winreg_HKEYType_Detach(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Detach_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___enter____doc__, "__enter__($self, /)\n" "--\n" @@ -77,6 +87,10 @@ winreg_HKEYType___enter__(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___exit____doc__, "__exit__($self, /, exc_type, exc_value, traceback)\n" "--\n" @@ -136,6 +150,10 @@ winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t n return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CloseKey__doc__, "CloseKey($module, hkey, /)\n" "--\n" @@ -151,6 +169,10 @@ PyDoc_STRVAR(winreg_CloseKey__doc__, #define WINREG_CLOSEKEY_METHODDEF \ {"CloseKey", (PyCFunction)winreg_CloseKey, METH_O, winreg_CloseKey__doc__}, +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_ConnectRegistry__doc__, "ConnectRegistry($module, computer_name, key, /)\n" "--\n" @@ -213,6 +235,10 @@ winreg_ConnectRegistry(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKey__doc__, "CreateKey($module, key, sub_key, /)\n" "--\n" @@ -278,6 +304,10 @@ winreg_CreateKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKeyEx__doc__, "CreateKeyEx($module, /, key, sub_key, reserved=0,\n" " access=winreg.KEY_WRITE)\n" @@ -398,6 +428,10 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKey__doc__, "DeleteKey($module, key, sub_key, /)\n" "--\n" @@ -452,6 +486,10 @@ winreg_DeleteKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKeyEx__doc__, "DeleteKeyEx($module, /, key, sub_key, access=winreg.KEY_WOW64_64KEY,\n" " reserved=0)\n" @@ -565,6 +603,10 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteValue__doc__, "DeleteValue($module, key, value, /)\n" "--\n" @@ -617,6 +659,10 @@ winreg_DeleteValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumKey__doc__, "EnumKey($module, key, index, /)\n" "--\n" @@ -661,6 +707,10 @@ winreg_EnumKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumValue__doc__, "EnumValue($module, key, index, /)\n" "--\n" @@ -714,6 +764,10 @@ winreg_EnumValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_ExpandEnvironmentStrings__doc__, "ExpandEnvironmentStrings($module, string, /)\n" "--\n" @@ -750,6 +804,10 @@ winreg_ExpandEnvironmentStrings(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_FlushKey__doc__, "FlushKey($module, key, /)\n" "--\n" @@ -790,6 +848,10 @@ winreg_FlushKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_LoadKey__doc__, "LoadKey($module, key, sub_key, file_name, /)\n" "--\n" @@ -866,6 +928,10 @@ winreg_LoadKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKey__doc__, "OpenKey($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -979,6 +1045,10 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKeyEx__doc__, "OpenKeyEx($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -1092,6 +1162,10 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryInfoKey__doc__, "QueryInfoKey($module, key, /)\n" "--\n" @@ -1128,6 +1202,10 @@ winreg_QueryInfoKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValue__doc__, "QueryValue($module, key, sub_key, /)\n" "--\n" @@ -1189,6 +1267,10 @@ winreg_QueryValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValueEx__doc__, "QueryValueEx($module, key, name, /)\n" "--\n" @@ -1246,6 +1328,10 @@ winreg_QueryValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_SaveKey__doc__, "SaveKey($module, key, file_name, /)\n" "--\n" @@ -1303,6 +1389,10 @@ winreg_SaveKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValue__doc__, "SetValue($module, key, sub_key, type, value, /)\n" "--\n" @@ -1384,6 +1474,10 @@ winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValueEx__doc__, "SetValueEx($module, key, value_name, reserved, type, value, /)\n" "--\n" @@ -1478,6 +1572,10 @@ winreg_SetValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_DisableReflectionKey__doc__, "DisableReflectionKey($module, key, /)\n" "--\n" @@ -1514,6 +1612,10 @@ winreg_DisableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_EnableReflectionKey__doc__, "EnableReflectionKey($module, key, /)\n" "--\n" @@ -1548,6 +1650,10 @@ winreg_EnableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_QueryReflectionKey__doc__, "QueryReflectionKey($module, key, /)\n" "--\n" @@ -1579,4 +1685,114 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7e817dc5edc914d3 input=a9049054013a1b77]*/ + +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#ifndef WINREG_HKEYTYPE_CLOSE_METHODDEF + #define WINREG_HKEYTYPE_CLOSE_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_CLOSE_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE_DETACH_METHODDEF + #define WINREG_HKEYTYPE_DETACH_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_DETACH_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___ENTER___METHODDEF + #define WINREG_HKEYTYPE___ENTER___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___ENTER___METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___EXIT___METHODDEF + #define WINREG_HKEYTYPE___EXIT___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___EXIT___METHODDEF) */ + +#ifndef WINREG_CLOSEKEY_METHODDEF + #define WINREG_CLOSEKEY_METHODDEF +#endif /* !defined(WINREG_CLOSEKEY_METHODDEF) */ + +#ifndef WINREG_CONNECTREGISTRY_METHODDEF + #define WINREG_CONNECTREGISTRY_METHODDEF +#endif /* !defined(WINREG_CONNECTREGISTRY_METHODDEF) */ + +#ifndef WINREG_CREATEKEY_METHODDEF + #define WINREG_CREATEKEY_METHODDEF +#endif /* !defined(WINREG_CREATEKEY_METHODDEF) */ + +#ifndef WINREG_CREATEKEYEX_METHODDEF + #define WINREG_CREATEKEYEX_METHODDEF +#endif /* !defined(WINREG_CREATEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEKEY_METHODDEF + #define WINREG_DELETEKEY_METHODDEF +#endif /* !defined(WINREG_DELETEKEY_METHODDEF) */ + +#ifndef WINREG_DELETEKEYEX_METHODDEF + #define WINREG_DELETEKEYEX_METHODDEF +#endif /* !defined(WINREG_DELETEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEVALUE_METHODDEF + #define WINREG_DELETEVALUE_METHODDEF +#endif /* !defined(WINREG_DELETEVALUE_METHODDEF) */ + +#ifndef WINREG_ENUMKEY_METHODDEF + #define WINREG_ENUMKEY_METHODDEF +#endif /* !defined(WINREG_ENUMKEY_METHODDEF) */ + +#ifndef WINREG_ENUMVALUE_METHODDEF + #define WINREG_ENUMVALUE_METHODDEF +#endif /* !defined(WINREG_ENUMVALUE_METHODDEF) */ + +#ifndef WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF + #define WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF +#endif /* !defined(WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF) */ + +#ifndef WINREG_FLUSHKEY_METHODDEF + #define WINREG_FLUSHKEY_METHODDEF +#endif /* !defined(WINREG_FLUSHKEY_METHODDEF) */ + +#ifndef WINREG_LOADKEY_METHODDEF + #define WINREG_LOADKEY_METHODDEF +#endif /* !defined(WINREG_LOADKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEY_METHODDEF + #define WINREG_OPENKEY_METHODDEF +#endif /* !defined(WINREG_OPENKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEYEX_METHODDEF + #define WINREG_OPENKEYEX_METHODDEF +#endif /* !defined(WINREG_OPENKEYEX_METHODDEF) */ + +#ifndef WINREG_QUERYINFOKEY_METHODDEF + #define WINREG_QUERYINFOKEY_METHODDEF +#endif /* !defined(WINREG_QUERYINFOKEY_METHODDEF) */ + +#ifndef WINREG_QUERYVALUE_METHODDEF + #define WINREG_QUERYVALUE_METHODDEF +#endif /* !defined(WINREG_QUERYVALUE_METHODDEF) */ + +#ifndef WINREG_QUERYVALUEEX_METHODDEF + #define WINREG_QUERYVALUEEX_METHODDEF +#endif /* !defined(WINREG_QUERYVALUEEX_METHODDEF) */ + +#ifndef WINREG_SAVEKEY_METHODDEF + #define WINREG_SAVEKEY_METHODDEF +#endif /* !defined(WINREG_SAVEKEY_METHODDEF) */ + +#ifndef WINREG_SETVALUE_METHODDEF + #define WINREG_SETVALUE_METHODDEF +#endif /* !defined(WINREG_SETVALUE_METHODDEF) */ + +#ifndef WINREG_SETVALUEEX_METHODDEF + #define WINREG_SETVALUEEX_METHODDEF +#endif /* !defined(WINREG_SETVALUEEX_METHODDEF) */ + +#ifndef WINREG_DISABLEREFLECTIONKEY_METHODDEF + #define WINREG_DISABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_DISABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_ENABLEREFLECTIONKEY_METHODDEF + #define WINREG_ENABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_ENABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF + #define WINREG_QUERYREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ +/*[clinic end generated code: output=715db416dc1321ee input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index b1481d79e6508d0..9d0fe6f87df69a7 100644 --- a/PC/config.c +++ b/PC/config.c @@ -43,10 +43,14 @@ extern PyObject* PyInit__collections(void); extern PyObject* PyInit__heapq(void); extern PyObject* PyInit__bisect(void); extern PyObject* PyInit__symtable(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_mmap(void); +#endif extern PyObject* PyInit__csv(void); extern PyObject* PyInit__sre(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__struct(void); extern PyObject* PyInit__datetime(void); extern PyObject* PyInit__functools(void); @@ -122,10 +126,14 @@ struct _inittab _PyImport_Inittab[] = { {"itertools", PyInit_itertools}, {"_collections", PyInit__collections}, {"_symtable", PyInit__symtable}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) {"mmap", PyInit_mmap}, +#endif {"_csv", PyInit__csv}, {"_sre", PyInit__sre}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif {"_struct", PyInit__struct}, {"_datetime", PyInit__datetime}, {"_functools", PyInit__functools}, diff --git a/PC/config_minimal.c b/PC/config_minimal.c index 928a4efd32e132f..9a66ea1d1cd3482 100644 --- a/PC/config_minimal.c +++ b/PC/config_minimal.c @@ -5,8 +5,10 @@ #include "Python.h" +#ifdef Py_ENABLE_SHARED /* Define extern variables omitted from minimal builds */ void *PyWin_DLLhModule = NULL; +#endif extern PyObject* PyInit_faulthandler(void); @@ -14,7 +16,9 @@ extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); extern PyObject* PyInit_nt(void); extern PyObject* PyInit__signal(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__ast(void); extern PyObject* PyInit__io(void); @@ -35,7 +39,9 @@ struct _inittab _PyImport_Inittab[] = { {"_tokenize", PyInit__tokenize}, {"_tracemalloc", PyInit__tracemalloc}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif /* This module "lives in" with marshal.c */ {"marshal", PyMarshal_Init}, diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 988d9c95aaa22ea..face4d03af9d4f6 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -253,6 +253,8 @@ msvcrt_getch_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwch -> wchar_t @@ -271,6 +273,8 @@ msvcrt_getwch_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.getche -> byte_char @@ -289,6 +293,8 @@ msvcrt_getche_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwche -> wchar_t @@ -307,6 +313,8 @@ msvcrt_getwche_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.putch @@ -326,6 +334,8 @@ msvcrt_putch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.putwch @@ -346,6 +356,8 @@ msvcrt_putwch_impl(PyObject *module, int unicode_char) } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.ungetch @@ -374,6 +386,8 @@ msvcrt_ungetch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.ungetwch @@ -398,6 +412,8 @@ msvcrt_ungetwch_impl(PyObject *module, int unicode_char) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP */ + #ifdef _DEBUG /*[clinic input] msvcrt.CrtSetReportFile -> HANDLE @@ -475,6 +491,8 @@ msvcrt_set_error_mode_impl(PyObject *module, int mode) } #endif /* _DEBUG */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] msvcrt.GetErrorMode @@ -494,6 +512,8 @@ msvcrt_GetErrorMode_impl(PyObject *module) return PyLong_FromUnsignedLong(res); } +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ + /*[clinic input] msvcrt.SetErrorMode @@ -601,10 +621,12 @@ PyInit_msvcrt(void) insertint(d, "LK_NBRLCK", _LK_NBRLCK); insertint(d, "LK_RLCK", _LK_RLCK); insertint(d, "LK_UNLCK", _LK_UNLCK); +#ifdef MS_WINDOWS_DESKTOP insertint(d, "SEM_FAILCRITICALERRORS", SEM_FAILCRITICALERRORS); insertint(d, "SEM_NOALIGNMENTFAULTEXCEPT", SEM_NOALIGNMENTFAULTEXCEPT); insertint(d, "SEM_NOGPFAULTERRORBOX", SEM_NOGPFAULTERRORBOX); insertint(d, "SEM_NOOPENFILEERRORBOX", SEM_NOOPENFILEERRORBOX); +#endif #ifdef _DEBUG insertint(d, "CRT_WARN", _CRT_WARN); insertint(d, "CRT_ERROR", _CRT_ERROR); diff --git a/PC/pyconfig.h b/PC/pyconfig.h index a34d420ab7ecaab..8a3bf8968ce29de 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -72,9 +72,27 @@ WIN32 is still required for the locale module. #define USE_SOCKET #endif -#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP) -#define MS_WINDOWS_NON_DESKTOP +#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) || defined(Py_BUILD_CORE_MODULE) +#include + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define MS_WINDOWS_DESKTOP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +#define MS_WINDOWS_APP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_SYSTEM) +#define MS_WINDOWS_SYSTEM #endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_GAMES) +#define MS_WINDOWS_GAMES +#endif + +/* Define to 1 if you support windows console io */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) +#define HAVE_WINDOWS_CONSOLE_IO 1 +#endif +#endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Compiler specific defines */ @@ -300,7 +318,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ -#if defined(MS_WIN64) +#ifdef MS_WIN64 /* maintain "win32" sys.platform for backward compatibility of Python code, the Win64 API should be close enough to the Win32 API to make this preferable */ diff --git a/PC/winreg.c b/PC/winreg.c index 86efed09855b01b..073598a12a68aaa 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -18,6 +18,8 @@ #include "structmember.h" // PyMemberDef #include +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) + static BOOL PyHKEY_AsHKEY(PyObject *ob, HKEY *pRes, BOOL bNoneOK); static BOOL clinic_HKEY_converter(PyObject *ob, void *p); static PyObject *PyHKEY_FromHKEY(HKEY h); @@ -829,6 +831,8 @@ winreg_CloseKey(PyObject *module, PyObject *hkey) Py_RETURN_NONE; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.ConnectRegistry -> HKEY @@ -866,6 +870,8 @@ winreg_ConnectRegistry_impl(PyObject *module, return retKey; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.CreateKey -> HKEY @@ -1272,6 +1278,8 @@ winreg_ExpandEnvironmentStrings_impl(PyObject *module, return o; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.FlushKey @@ -1305,6 +1313,9 @@ winreg_FlushKey_impl(PyObject *module, HKEY key) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) /*[clinic input] winreg.LoadKey @@ -1354,6 +1365,8 @@ winreg_LoadKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.OpenKey -> HKEY @@ -1463,6 +1476,7 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) return ret; } + /*[clinic input] winreg.QueryValue @@ -1634,6 +1648,8 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const Py_UNICODE *name) return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.SaveKey @@ -1679,6 +1695,8 @@ winreg_SaveKey_impl(PyObject *module, HKEY key, const Py_UNICODE *file_name) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.SetValue @@ -1776,6 +1794,7 @@ winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, return result; } + /*[clinic input] winreg.SetValueEx @@ -1861,6 +1880,8 @@ winreg_SetValueEx_impl(PyObject *module, HKEY key, return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.DisableReflectionKey @@ -2009,6 +2030,8 @@ winreg_QueryReflectionKey_impl(PyObject *module, HKEY key) return PyBool_FromLong(result); } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + static struct PyMethodDef winreg_methods[] = { WINREG_CLOSEKEY_METHODDEF WINREG_CONNECTREGISTRY_METHODDEF @@ -2149,3 +2172,5 @@ PyMODINIT_FUNC PyInit_winreg(void) ADD_INT(REG_RESOURCE_REQUIREMENTS_LIST); return m; } + +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM || MS_WINDOWS_GAMES */ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index d55fcefbb6f206e..3f0e29f051a4387 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -13,7 +13,9 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS +# ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN +# endif # include "windows.h" #endif /* MS_WINDOWS */ @@ -108,7 +110,7 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) /* NOTREACHED */ } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Readline implementation using ReadConsoleW */ extern char _get_console_type(HANDLE handle); @@ -233,7 +235,7 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) return buf; } -#endif +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Readline implementation using fgets() */ @@ -246,7 +248,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) PyThreadState *tstate = _PyOS_ReadlineTState; assert(tstate != NULL); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (!config->legacy_windows_stdio && sys_stdin == stdin) { HANDLE hStdIn, hStdErr; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 7bd04d573df4adf..acab05e2c6def3a 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -163,6 +163,7 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } +#ifdef Py_ENABLE_SHARED /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -216,6 +217,7 @@ _Py_CheckPython3(void) return hPython3 != NULL; #undef MAXPATHLEN } +#endif /* Py_ENABLE_SHARED */ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, @@ -224,7 +226,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; +#ifdef Py_ENABLE_SHARED _Py_CheckPython3(); +#endif /* Py_ENABLE_SHARED */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) @@ -234,10 +238,12 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, { HINSTANCE hDLL = NULL; +#ifdef MS_WINDOWS_DESKTOP unsigned int old_mode; /* Don't display a message box when Python can't load a DLL */ old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); +#endif /* bpo-36085: We use LoadLibraryEx with restricted search paths to avoid DLL preloading attacks and enable use of the @@ -250,8 +256,10 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, Py_END_ALLOW_THREADS PyMem_Free(wpathname); +#ifdef MS_WINDOWS_DESKTOP /* restore old error mode settings */ SetErrorMode(old_mode); +#endif if (hDLL==NULL){ PyObject *message; diff --git a/Python/fileutils.c b/Python/fileutils.c index 93bee9ee007cd4d..4ac759c45a3a1ee 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,7 +8,11 @@ #ifdef MS_WINDOWS # include # include -# include // PathCchCombineEx +# if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +# define PATHCCH_ALLOW_LONG_PATHS 0x01 +# else +# include // PathCchCombineEx +# endif extern int winerror_to_errno(int); #endif @@ -77,7 +81,8 @@ _Py_device_encoding(int fd) if (!valid) Py_RETURN_NONE; -#if defined(MS_WINDOWS) +#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO UINT cp; if (fd == 0) cp = GetConsoleCP(); @@ -92,6 +97,9 @@ _Py_device_encoding(int fd) } return PyUnicode_FromFormat("cp%u", (unsigned int)cp); +#else + Py_RETURN_NONE; +#endif /* HAVE_WINDOWS_CONSOLE_IO */ #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); @@ -1270,6 +1278,13 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#ifdef MS_WINDOWS +// For some Windows API partitions, SetHandleInformation() is declared +// but none of the handle flags are defined. +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif +#endif /* This function MUST be kept async-signal-safe on POSIX when raise=0. */ static int @@ -2096,6 +2111,72 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) #endif } +// The Windows Games API family implements the PathCch* APIs in the Xbox OS, +// but does not expose them yet. Load them dynamically until +// 1) they are officially exposed +// 2) we stop supporting older versions of the GDK which do not expose them +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +HRESULT +PathCchSkipRoot(const wchar_t *path, const wchar_t **rootEnd) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchSkipRoot) (PCWSTR pszPath, + PCWSTR *ppszRootEnd); + static PPathCchSkipRoot _PathCchSkipRoot; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchSkipRoot = (PPathCchSkipRoot)GetProcAddress( + pathapi, "PathCchSkipRoot"); + } + else { + _PathCchSkipRoot = NULL; + } + initialized = 1; + } + + if (!_PathCchSkipRoot) { + return E_NOINTERFACE; + } + + return _PathCchSkipRoot(path, rootEnd); +} + +static HRESULT +PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, + const wchar_t *relfile, unsigned long flags) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, + size_t cchPathOut, + PCWSTR pszPathIn, + PCWSTR pszMore, + unsigned long dwFlags); + static PPathCchCombineEx _PathCchCombineEx; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress( + pathapi, "PathCchCombineEx"); + } + else { + _PathCchCombineEx = NULL; + } + initialized = 1; + } + + if (!_PathCchCombineEx) { + return E_NOINTERFACE; + } + + return _PathCchCombineEx(buffer, bufsize, dirname, relfile, flags); +} + +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // The caller must ensure "buffer" is big enough. static int @@ -2491,12 +2572,12 @@ _Py_get_blocking(int fd) success = GetNamedPipeHandleStateW(handle, &mode, NULL, NULL, NULL, NULL, 0); Py_END_ALLOW_THREADS - + if (!success) { PyErr_SetFromWindowsErr(0); return -1; } - + return !(mode & PIPE_NOWAIT); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e80dd30c89dfd0c..82e94090a6027ad 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2289,7 +2289,7 @@ create_stdio(const PyConfig *config, PyObject* io, raw = Py_NewRef(buf); } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Windows console IO is always UTF-8 encoded */ PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 207abb964bcac93..764fb70bae6c384 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1490,6 +1490,9 @@ static PyStructSequence_Desc windows_version_desc = { static PyObject * _sys_getwindowsversion_from_kernel32() { +#ifndef MS_WINDOWS_DESKTOP + return NULL; +#else HANDLE hKernel32; wchar_t kernel32_path[MAX_PATH]; LPVOID verblock; @@ -1523,6 +1526,7 @@ _sys_getwindowsversion_from_kernel32() realBuild = HIWORD(ffi->dwProductVersionLS); PyMem_RawFree(verblock); return Py_BuildValue("(kkk)", realMajor, realMinor, realBuild); +#endif /* !MS_WINDOWS_DESKTOP */ } /* Disable deprecation warnings about GetVersionEx as the result is From 71cf7c3dddd9c49ec70c1a95547f2fcd5daa7034 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 9 Mar 2023 23:06:20 +0000 Subject: [PATCH 21/35] gh-102378: don't bother stripping `/` from __text_signature__ (#102379) --- Lib/inspect.py | 57 ++++++------------- Lib/test/test_inspect.py | 23 +++----- ...-03-03-19-53-08.gh-issue-102378.kRdOZc.rst | 1 + 3 files changed, 25 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 166667c62cdccf8..edc23b0ffa9201f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2106,26 +2106,21 @@ def _signature_strip_non_python_syntax(signature): Private helper function. Takes a signature in Argument Clinic's extended signature format. - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, + Returns a tuple of two things: + * that signature re-rendered in standard Python syntax, and * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. + the function does not have a "self" parameter. """ if not signature: - return signature, None, None + return signature, None self_parameter = None - last_positional_only = None lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) - delayed_comma = False - skip_next_comma = False text = [] add = text.append @@ -2142,35 +2137,18 @@ def _signature_strip_non_python_syntax(signature): if type == OP: if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue - - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue + current_parameter += 1 if (type == ERRORTOKEN) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') add(string) if (string == ','): add(' ') clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only + return clean_signature, self_parameter def _signature_fromstr(cls, obj, s, skip_bound_arg=True): @@ -2179,8 +2157,7 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """ Parameter = cls._parameter_cls - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) + clean_signature, self_parameter = _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" @@ -2269,17 +2246,17 @@ def p(name_node, default_node, default=empty): parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): + total_non_kw_args = len(f.args.posonlyargs) + len(f.args.args) + required_non_kw_args = total_non_kw_args - len(f.args.defaults) + defaults = itertools.chain(itertools.repeat(None, required_non_kw_args), f.args.defaults) + + kind = Parameter.POSITIONAL_ONLY + for (name, default) in zip(f.args.posonlyargs, defaults): + p(name, default) + + kind = Parameter.POSITIONAL_OR_KEYWORD + for (name, default) in zip(f.args.args, defaults): p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD # *args if f.args.vararg: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 92aba519d28a085..02f8378d0413eac 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4230,56 +4230,47 @@ def foo(a): pass class TestSignaturePrivateHelpers(unittest.TestCase): def _strip_non_python_syntax(self, input, - clean_signature, self_parameter, last_positional_only): + clean_signature, self_parameter): computed_clean_signature, \ - computed_self_parameter, \ - computed_last_positional_only = \ + computed_self_parameter = \ inspect._signature_strip_non_python_syntax(input) self.assertEqual(computed_clean_signature, clean_signature) self.assertEqual(computed_self_parameter, self_parameter) - self.assertEqual(computed_last_positional_only, last_positional_only) def test_signature_strip_non_python_syntax(self): self._strip_non_python_syntax( "($module, /, path, mode, *, dir_fd=None, " + "effective_ids=False,\n follow_symlinks=True)", - "(module, path, mode, *, dir_fd=None, " + + "(module, /, path, mode, *, dir_fd=None, " + "effective_ids=False, follow_symlinks=True)", - 0, 0) self._strip_non_python_syntax( "($module, word, salt, /)", - "(module, word, salt)", - 0, - 2) + "(module, word, salt, /)", + 0) self._strip_non_python_syntax( "(x, y=None, z=None, /)", - "(x, y=None, z=None)", - None, - 2) + "(x, y=None, z=None, /)", + None) self._strip_non_python_syntax( "(x, y=None, z=None)", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "(x,\n y=None,\n z = None )", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "", "", - None, None) self._strip_non_python_syntax( - None, None, None, None) diff --git a/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst new file mode 100644 index 000000000000000..d30f65f30d109aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst @@ -0,0 +1 @@ +Private helper method ``inspect._signature_strip_non_python_syntax`` will no longer strip ``/`` from the input string. From 2999e02836f9112de6b17784eaca762fb87e71a9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:02:32 +0000 Subject: [PATCH 22/35] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives in `_ctypes` (#102477) --- Modules/_ctypes/_ctypes.c | 11 ++--------- Modules/_ctypes/callproc.c | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8690f2c1b078524..6f92ca08dd537b7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2200,7 +2200,6 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) struct fielddesc *fd; PyObject *as_parameter; int res; - PyObject *exc, *val, *tb; /* If the value is already an instance of the requested type, we can use it as is */ @@ -2234,33 +2233,27 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj) return (PyObject *)parg; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); Py_DECREF(parg); if (_PyObject_LookupAttr(value, &_Py_ID(_as_parameter_), &as_parameter) < 0) { Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } if (as_parameter) { if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } value = PyCSimpleType_from_param(type, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return value; } if (exc) { - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); } else { PyErr_SetString(PyExc_TypeError, "wrong type"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f6d98bbeebc24e9..4438727332bc11a 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1013,41 +1013,43 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) { va_list vargs; - PyObject *tp, *v, *tb, *s, *cls_str, *msg_str; va_start(vargs, fmt); - s = PyUnicode_FromFormatV(fmt, vargs); + PyObject *s = PyUnicode_FromFormatV(fmt, vargs); va_end(vargs); - if (!s) + if (s == NULL) { return; + } - PyErr_Fetch(&tp, &v, &tb); - PyErr_NormalizeException(&tp, &v, &tb); - if (PyType_Check(tp)) - cls_str = PyType_GetName((PyTypeObject *)tp); - else - cls_str = PyObject_Str(tp); + assert(PyErr_Occurred()); + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + PyObject *cls_str = PyType_GetName(Py_TYPE(exc)); if (cls_str) { PyUnicode_AppendAndDel(&s, cls_str); PyUnicode_AppendAndDel(&s, PyUnicode_FromString(": ")); - if (s == NULL) + if (s == NULL) { goto error; - } else + } + } + else { PyErr_Clear(); - msg_str = PyObject_Str(v); - if (msg_str) + } + + PyObject *msg_str = PyObject_Str(exc); + if (msg_str) { PyUnicode_AppendAndDel(&s, msg_str); + } else { PyErr_Clear(); PyUnicode_AppendAndDel(&s, PyUnicode_FromString("???")); } - if (s == NULL) + if (s == NULL) { goto error; + } PyErr_SetObject(exc_class, s); error: - Py_XDECREF(tp); - Py_XDECREF(v); - Py_XDECREF(tb); + Py_XDECREF(exc); Py_XDECREF(s); } From cb35882773a3ffc7fe0671e64848f4c926a2d52f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 12:21:37 +0000 Subject: [PATCH 23/35] gh-102519: Add os.listdrives, os.listvolumes and os.listmounts on Windows (GH-102544) --- Doc/whatsnew/3.12.rst | 4 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Lib/test/test_os.py | 43 ++++ ...-03-08-23-08-38.gh-issue-102519.wlcsFI.rst | 2 + Modules/clinic/posixmodule.c.h | 128 +++++++++++- Modules/posixmodule.c | 194 ++++++++++++++++++ 9 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d982cb62ec2f4e3..48b7aab0595ebb2 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -294,6 +294,10 @@ os method to check if the entry is a junction. (Contributed by Charles Machalow in :gh:`99547`.) +* Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` + functions on Windows for enumerating drives, volumes and mount points. + (Contributed by Steve Dower in :gh:`102519`.) + os.path ------- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index dc5cd58d853534b..4b12ae523c32601 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1216,6 +1216,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8b23aa154793018..17fb9ffbbf9f112 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -702,6 +702,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) + STRUCT_FOR_ID(volume) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 471efadb13bb4f4..b240be57369d9df 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1208,6 +1208,7 @@ extern "C" { INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ + INIT_ID(volume), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b47d240e492ff97..52af37a8e60aa8e 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1310,6 +1310,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(version); PyUnicode_InternInPlace(&string); + string = &_Py_ID(volume); + PyUnicode_InternInPlace(&string); string = &_Py_ID(warnings); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnoptions); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ba6feb69ea17125..253e2a23238f12a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2648,6 +2648,49 @@ def test_listdir_extended_path(self): [os.fsencode(path) for path in self.created_paths]) +@unittest.skipUnless(os.name == "nt", "NT specific tests") +class Win32ListdriveTests(unittest.TestCase): + """Test listdrive, listmounts and listvolume on Windows.""" + + def setUp(self): + # Get drives and volumes from fsutil + out = subprocess.check_output( + ["fsutil.exe", "volume", "list"], + cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), + encoding="mbcs", + errors="ignore", + ) + lines = out.splitlines() + self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} + self.known_drives = {l for l in lines if l[1:] == ':\\'} + self.known_mounts = {l for l in lines if l[1:3] == ':\\'} + + def test_listdrives(self): + drives = os.listdrives() + self.assertIsInstance(drives, list) + self.assertSetEqual( + self.known_drives, + self.known_drives & set(drives), + ) + + def test_listvolumes(self): + volumes = os.listvolumes() + self.assertIsInstance(volumes, list) + self.assertSetEqual( + self.known_volumes, + self.known_volumes & set(volumes), + ) + + def test_listmounts(self): + for volume in os.listvolumes(): + mounts = os.listmounts(volume) + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) + + @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') class ReadlinkTests(unittest.TestCase): filelink = 'readlinktest' diff --git a/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst new file mode 100644 index 000000000000000..f47e4f70b1301d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst @@ -0,0 +1,2 @@ +Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` +functions on Windows for enumerating drives, volumes and mount points diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6565f8df935cb85..8b0550d832fc0aa 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1601,6 +1601,120 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #if defined(MS_WINDOWS) +PyDoc_STRVAR(os_listdrives__doc__, +"listdrives($module, /)\n" +"--\n" +"\n" +"Return a list containing the names of drives in the system.\n" +"\n" +"A drive name typically looks like \'C:\\\\\'."); + +#define OS_LISTDRIVES_METHODDEF \ + {"listdrives", (PyCFunction)os_listdrives, METH_NOARGS, os_listdrives__doc__}, + +static PyObject * +os_listdrives_impl(PyObject *module); + +static PyObject * +os_listdrives(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listdrives_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listvolumes__doc__, +"listvolumes($module, /)\n" +"--\n" +"\n" +"Return a list containing the volumes in the system.\n" +"\n" +"Volumes are typically represented as a GUID path."); + +#define OS_LISTVOLUMES_METHODDEF \ + {"listvolumes", (PyCFunction)os_listvolumes, METH_NOARGS, os_listvolumes__doc__}, + +static PyObject * +os_listvolumes_impl(PyObject *module); + +static PyObject * +os_listvolumes(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listvolumes_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listmounts__doc__, +"listmounts($module, /, volume)\n" +"--\n" +"\n" +"Return a list containing mount points for a particular volume.\n" +"\n" +"\'volume\' should be a GUID path as returned from os.listvolumes."); + +#define OS_LISTMOUNTS_METHODDEF \ + {"listmounts", _PyCFunction_CAST(os_listmounts), METH_FASTCALL|METH_KEYWORDS, os_listmounts__doc__}, + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume); + +static PyObject * +os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(volume), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"volume", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "listmounts", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &volume)) { + goto exit; + } + return_value = os_listmounts_impl(module, &volume); + +exit: + /* Cleanup for volume */ + path_cleanup(&volume); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__getfullpathname__doc__, "_getfullpathname($module, path, /)\n" "--\n" @@ -11253,6 +11367,18 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_LINK_METHODDEF #endif /* !defined(OS_LINK_METHODDEF) */ +#ifndef OS_LISTDRIVES_METHODDEF + #define OS_LISTDRIVES_METHODDEF +#endif /* !defined(OS_LISTDRIVES_METHODDEF) */ + +#ifndef OS_LISTVOLUMES_METHODDEF + #define OS_LISTVOLUMES_METHODDEF +#endif /* !defined(OS_LISTVOLUMES_METHODDEF) */ + +#ifndef OS_LISTMOUNTS_METHODDEF + #define OS_LISTMOUNTS_METHODDEF +#endif /* !defined(OS_LISTMOUNTS_METHODDEF) */ + #ifndef OS__GETFULLPATHNAME_METHODDEF #define OS__GETFULLPATHNAME_METHODDEF #endif /* !defined(OS__GETFULLPATHNAME_METHODDEF) */ @@ -11796,4 +11922,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47750e0e29c8d707 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0d534f35f6a420a..7d91f7e4bac76b9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4229,7 +4229,198 @@ os_listdir_impl(PyObject *module, path_t *path) #endif } + #ifdef MS_WINDOWS + +/*[clinic input] +os.listdrives + +Return a list containing the names of drives in the system. + +A drive name typically looks like 'C:\\'. + +[clinic start generated code]*/ + +static PyObject * +os_listdrives_impl(PyObject *module) +/*[clinic end generated code: output=aaece9dacdf682b5 input=1af9ccc9e583798e]*/ +{ + /* Number of possible drives is limited, so 256 should always be enough. + On the day when it is not, listmounts() will have to be used. */ + wchar_t buffer[256]; + DWORD buflen = Py_ARRAY_LENGTH(buffer); + PyObject *result = NULL; + if (PySys_Audit("os.listdrives", NULL) < 0) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + buflen = GetLogicalDriveStringsW(buflen, buffer); + Py_END_ALLOW_THREADS; + + if (!buflen) { + PyErr_SetFromWindowsErr(0); + return NULL; + } else if (buflen >= Py_ARRAY_LENGTH(buffer)) { + PyErr_SetFromWindowsErr(ERROR_MORE_DATA); + return NULL; + } + + /* buflen includes a null terminator, so remove it */ + PyObject *str = PyUnicode_FromWideChar(buffer, buflen - 1); + if (str) { + PyObject *nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + Py_DECREF(nullchar); + } + Py_DECREF(str); + } + return result; +} + +/*[clinic input] +os.listvolumes + +Return a list containing the volumes in the system. + +Volumes are typically represented as a GUID path. + +[clinic start generated code]*/ + +static PyObject * +os_listvolumes_impl(PyObject *module) +/*[clinic end generated code: output=534e10ea2bf9d386 input=f6e4e70371f11e99]*/ +{ + PyObject *result = PyList_New(0); + HANDLE find = INVALID_HANDLE_VALUE; + wchar_t buffer[MAX_PATH + 1]; + if (!result) { + return NULL; + } + if (PySys_Audit("os.listvolumes", NULL) < 0) { + Py_DECREF(result); + return NULL; + } + + int err = 0; + Py_BEGIN_ALLOW_THREADS; + find = FindFirstVolumeW(buffer, Py_ARRAY_LENGTH(buffer)); + if (find == INVALID_HANDLE_VALUE) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + + while (!err) { + PyObject *s = PyUnicode_FromWideChar(buffer, -1); + if (!s || PyList_Append(result, s) < 0) { + Py_XDECREF(s); + Py_CLEAR(result); + break; + } + Py_DECREF(s); + + Py_BEGIN_ALLOW_THREADS; + if (!FindNextVolumeW(find, buffer, Py_ARRAY_LENGTH(buffer))) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + } + + if (find != INVALID_HANDLE_VALUE) { + Py_BEGIN_ALLOW_THREADS; + FindVolumeClose(find); + Py_END_ALLOW_THREADS; + } + if (err && err != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(err); + Py_XDECREF(result); + result = NULL; + } + return result; +} + + +/*[clinic input] +os.listmounts + + volume: path_t + +Return a list containing mount points for a particular volume. + +'volume' should be a GUID path as returned from os.listvolumes. + +[clinic start generated code]*/ + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume) +/*[clinic end generated code: output=06da49679de4512e input=a8a27178e3f67845]*/ +{ + wchar_t default_buffer[MAX_PATH + 1]; + DWORD buflen = Py_ARRAY_LENGTH(default_buffer); + LPWSTR buffer = default_buffer; + DWORD attributes; + PyObject *str = NULL; + PyObject *nullchar = NULL; + PyObject *result = NULL; + + /* Ensure we have a valid volume path before continuing */ + Py_BEGIN_ALLOW_THREADS + attributes = GetFileAttributesW(volume->wide); + Py_END_ALLOW_THREADS + if (attributes == INVALID_FILE_ATTRIBUTES && + GetLastError() == ERROR_UNRECOGNIZED_VOLUME) + { + return PyErr_SetFromWindowsErr(ERROR_UNRECOGNIZED_VOLUME); + } + + if (PySys_Audit("os.listmounts", "O", volume->object) < 0) { + return NULL; + } + + while (1) { + BOOL success; + Py_BEGIN_ALLOW_THREADS + success = GetVolumePathNamesForVolumeNameW(volume->wide, buffer, + buflen, &buflen); + Py_END_ALLOW_THREADS + if (success) { + break; + } + if (GetLastError() != ERROR_MORE_DATA) { + PyErr_SetFromWindowsErr(0); + goto exit; + } + if (buffer != default_buffer) { + PyMem_Free((void *)buffer); + } + buffer = (wchar_t*)PyMem_Malloc(sizeof(wchar_t) * buflen); + if (!buffer) { + PyErr_NoMemory(); + goto exit; + } + } + if (buflen < 2) { + result = PyList_New(0); + goto exit; + } + // buflen includes two null terminators, one for the last string + // and one for the array of strings. + str = PyUnicode_FromWideChar(buffer, buflen - 2); + nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (str && nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + } +exit: + if (buffer != default_buffer) { + PyMem_Free(buffer); + } + Py_XDECREF(nullchar); + Py_XDECREF(str); + return result; +} + + int _PyOS_getfullpathname(const wchar_t *path, wchar_t **abspath_p) { @@ -15252,6 +15443,9 @@ static PyMethodDef posix_methods[] = { OS_GETCWDB_METHODDEF OS_LINK_METHODDEF OS_LISTDIR_METHODDEF + OS_LISTDRIVES_METHODDEF + OS_LISTMOUNTS_METHODDEF + OS_LISTVOLUMES_METHODDEF OS_LSTAT_METHODDEF OS_MKDIR_METHODDEF OS_NICE_METHODDEF From 53dceb53ade15587b9cfd30c0a0942232517dee9 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Fri, 10 Mar 2023 19:22:02 +0700 Subject: [PATCH 24/35] gh-86509: Add link to Lib/_threading_local.py in threading docs (#101824) --- Doc/library/threading.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b352125551fa79e..83ed48052704fb9 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -272,7 +272,7 @@ The instance's values will be different for separate threads. A class that represents thread-local data. For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module. + :mod:`_threading_local` module: :source:`Lib/_threading_local.py`. .. _thread-objects: From 64bde502cf89963bc7382b03ea9e1c0967d22e35 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 10 Mar 2023 10:29:37 -0500 Subject: [PATCH 25/35] GH-102537: Handle check for PYTHONTZPATH failing in zoneinfo test (GH-102538) It is possible but unlikely for the `python_tzpath_context` function to fail between the start of the `try` block and the point where `os.environ.get` succeeds, in which case `old_env` will be undefined. In this case, we want to take no action. Practically speaking this will really only happen in an error condition anyway, so it doesn't really matter, but we should probably do it right anyway. --- Lib/test/test_zoneinfo/test_zoneinfo.py | 9 ++++++++- .../Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 82041a2b488334f..ae921f7432c4665 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1543,13 +1543,20 @@ class TzPathTest(TzPathUserMixin, ZoneInfoTestBase): @contextlib.contextmanager def python_tzpath_context(value): path_var = "PYTHONTZPATH" + unset_env_sentinel = object() + old_env = unset_env_sentinel try: with OS_ENV_LOCK: old_env = os.environ.get(path_var, None) os.environ[path_var] = value yield finally: - if old_env is None: + if old_env is unset_env_sentinel: + # In this case, `old_env` was never retrieved from the + # environment for whatever reason, so there's no need to + # reset the environment TZPATH. + pass + elif old_env is None: del os.environ[path_var] else: os.environ[path_var] = old_env # pragma: nocover diff --git a/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst new file mode 100644 index 000000000000000..94d160dd4127a6c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst @@ -0,0 +1,2 @@ +Adjust the error handling strategy in +``test_zoneinfo.TzPathTest.python_tzpath_context``. Patch by Paul Ganssle. From 12226bec2588f925f4698e1130ce78e118343934 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 15:41:32 +0000 Subject: [PATCH 26/35] gh-102519: Add doc updates for os.listdrives, listvolumes and listmounts (GH-102585) --- Doc/library/os.rst | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 23ce98785bedfc6..5b9f49be1fad55f 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2188,6 +2188,69 @@ features: Accepts a :term:`path-like object`. +.. function:: listdrives() + + Return a list containing the names of drives on a Windows system. + + A drive name typically looks like ``'C:\\'``. Not every drive name + will be associated with a volume, and some may be inaccessible for + a variety of reasons, including permissions, network connectivity + or missing media. This function does not test for access. + + May raise :exc:`OSError` if an error occurs collecting the drive + names. + + .. audit-event:: os.listdrives "" os.listdrives + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listmounts(volume) + + Return a list containing the mount points for a volume on a Windows + system. + + *volume* must be represented as a GUID path, like those returned by + :func:`os.listvolumes`. Volumes may be mounted in multiple locations + or not at all. In the latter case, the list will be empty. Mount + points that are not associated with a volume will not be returned by + this function. + + The mount points return by this function will be absolute paths, and + may be longer than the drive name. + + Raises :exc:`OSError` if the volume is not recognized or if an error + occurs collecting the paths. + + .. audit-event:: os.listmounts volume os.listmounts + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listvolumes() + + Return a list containing the volumes in the system. + + Volumes are typically represented as a GUID path that looks like + ``\\?\Volume{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}\``. Files can + usually be accessed through a GUID path, permissions allowing. + However, users are generally not familiar with them, and so the + recommended use of this function is to retrieve mount points + using :func:`os.listmounts`. + + May raise :exc:`OSError` if an error occurs collecting the volumes. + + .. audit-event:: os.listvolumes "" os.listvolumes + + .. availability:: Windows + + .. versionadded:: 3.12 + + .. function:: lstat(path, *, dir_fd=None) Perform the equivalent of an :c:func:`lstat` system call on the given path. From 90f1d777177e28b6c7b8d9ba751550e373d61b0a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 10 Mar 2023 17:29:04 +0000 Subject: [PATCH 27/35] GH-80486: Fix handling of NTFS alternate data streams in pathlib (GH-102454) Co-authored-by: Maor Kleinberger --- Lib/pathlib.py | 8 ++++-- Lib/test/test_pathlib.py | 28 ++++++++++++++++++- .../2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst | 2 ++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d375529ff5f7676..55c44f12e5a2fba 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -320,8 +320,9 @@ def _from_parsed_parts(cls, drv, root, parts): def _format_parsed_parts(cls, drv, root, parts): if drv or root: return drv + root + cls._flavour.sep.join(parts[1:]) - else: - return cls._flavour.sep.join(parts) + elif parts and cls._flavour.splitdrive(parts[0])[0]: + parts = ['.'] + parts + return cls._flavour.sep.join(parts) def __str__(self): """Return the string representation of the path, suitable for @@ -1188,7 +1189,8 @@ def expanduser(self): homedir = self._flavour.expanduser(self._parts[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") - return self._from_parts([homedir] + self._parts[1:]) + drv, root, parts = self._parse_parts((homedir,)) + return self._from_parsed_parts(drv, root, parts + self._parts[1:]) return self diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index f8e2f44d27fc1ef..f05dead5886743e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -122,6 +122,13 @@ def test_parse_parts(self): # the second path is relative. check(['c:/a/b', 'c:x/y'], ('c:', '\\', ['c:\\', 'a', 'b', 'x', 'y'])) check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) + # Paths to files with NTFS alternate data streams + check(['./c:s'], ('', '', ['c:s'])) + check(['cc:s'], ('', '', ['cc:s'])) + check(['C:c:s'], ('C:', '', ['C:', 'c:s'])) + check(['C:/c:s'], ('C:', '\\', ['C:\\', 'c:s'])) + check(['D:a', './c:b'], ('D:', '', ['D:', 'a', 'c:b'])) + check(['D:/a', './c:b'], ('D:', '\\', ['D:\\', 'a', 'c:b'])) # @@ -165,6 +172,7 @@ def test_constructor_common(self): self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) def test_bytes(self): P = self.cls @@ -814,7 +822,8 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): equivalences = _BasePurePathTest.equivalences.copy() equivalences.update({ - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('/', 'c:', 'a') ], + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), @@ -838,6 +847,7 @@ def test_str(self): self.assertEqual(str(p), '\\\\a\\b\\c\\d') def test_str_subclass(self): + self._check_str_subclass('.\\a:b') self._check_str_subclass('c:') self._check_str_subclass('c:a') self._check_str_subclass('c:a\\b.txt') @@ -1005,6 +1015,7 @@ def test_drive(self): self.assertEqual(P('//a/b').drive, '\\\\a\\b') self.assertEqual(P('//a/b/').drive, '\\\\a\\b') self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') def test_root(self): P = self.cls @@ -1341,6 +1352,14 @@ def test_join(self): self.assertEqual(pp, P('C:/a/b/x/y')) pp = p.joinpath('c:/x/y') self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) def test_div(self): # Basically the same as joinpath(). @@ -1361,6 +1380,11 @@ def test_div(self): # the second path is relative. self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) def test_is_reserved(self): P = self.cls @@ -1626,6 +1650,8 @@ def test_expanduser_common(self): self.assertEqual(p.expanduser(), p) p = P(P('').absolute().anchor) / '~' self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) def test_exists(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst new file mode 100644 index 000000000000000..d9360496ac24cb9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst @@ -0,0 +1,2 @@ +Fix handling of Windows filenames that resemble drives, such as ``./a:b``, +in :mod:`pathlib`. From ee6f8413a99d0ee4828e1c81911e203d3fff85d5 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 11 Mar 2023 08:21:22 +0800 Subject: [PATCH 28/35] gh-102578: Optimise setting and deleting mutable attributes on non-dataclass subclasses of frozen dataclasses (gh-102573) --- Lib/dataclasses.py | 10 ++--- Lib/test/test_dataclasses.py | 44 +++++++++++++++++++ ...-03-10-13-21-16.gh-issue-102578.-gujoI.rst | 4 ++ 3 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8bc8594d674bc0d..78a126f051e2e76 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -616,21 +616,19 @@ def _repr_fn(fields, globals): def _frozen_get_del_attr(cls, fields, globals): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} + condition = 'type(self) is cls' if fields: - fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' - else: - # Special case for the zero-length tuple. - fields_str = '()' + condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' return (_create_fn('__setattr__', ('self', 'name', 'value'), - (f'if type(self) is cls or name in {fields_str}:', + (f'if {condition}:', ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', f'super(cls, self).__setattr__(name, value)'), locals=locals, globals=globals), _create_fn('__delattr__', ('self', 'name'), - (f'if type(self) is cls or name in {fields_str}:', + (f'if {condition}:', ' raise FrozenInstanceError(f"cannot delete field {name!r}")', f'super(cls, self).__delattr__(name)'), locals=locals, diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 589f229f4623594..5486b2ef3f47e51 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2767,6 +2767,19 @@ class C: c.i = 5 self.assertEqual(c.i, 10) + def test_frozen_empty(self): + @dataclass(frozen=True) + class C: + pass + + c = C() + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + del c.i + def test_inherit(self): @dataclass(frozen=True) class C: @@ -2890,6 +2903,37 @@ class S(D): self.assertEqual(s.y, 10) self.assertEqual(s.cached, True) + with self.assertRaises(FrozenInstanceError): + del s.x + self.assertEqual(s.x, 3) + with self.assertRaises(FrozenInstanceError): + del s.y + self.assertEqual(s.y, 10) + del s.cached + self.assertFalse(hasattr(s, 'cached')) + with self.assertRaises(AttributeError) as cm: + del s.cached + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + + def test_non_frozen_normal_derived_from_empty_frozen(self): + @dataclass(frozen=True) + class D: + pass + + class S(D): + pass + + s = S() + self.assertFalse(hasattr(s, 'x')) + s.x = 5 + self.assertEqual(s.x, 5) + + del s.x + self.assertFalse(hasattr(s, 'x')) + with self.assertRaises(AttributeError) as cm: + del s.x + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + def test_overwriting_frozen(self): # frozen uses __setattr__ and __delattr__. with self.assertRaisesRegex(TypeError, diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst new file mode 100644 index 000000000000000..7307148d9a81efc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst @@ -0,0 +1,4 @@ +Speed up setting or deleting mutable attributes on non-dataclass subclasses of +frozen dataclasses. Due to the implementation of ``__setattr__`` and +``__delattr__`` for frozen dataclasses, this previously had a time complexity +of ``O(n)``. It now has a time complexity of ``O(1)``. From b48be8fa18518583abb21bf6e4f5d7e4b5c9d7b2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 11 Mar 2023 03:26:46 +0300 Subject: [PATCH 29/35] gh-102103: add `module` argument to `dataclasses.make_dataclass` (#102104) --- Doc/library/dataclasses.rst | 6 ++- Lib/dataclasses.py | 15 ++++++- Lib/test/test_dataclasses.py | 39 +++++++++++++++++++ ...-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst | 2 + 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 82faa7b77450fb6..5f4dc25bfd78771 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -389,7 +389,7 @@ Module contents :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) Creates a new dataclass with name ``cls_name``, fields as defined in ``fields``, base classes as given in ``bases``, and initialized @@ -401,6 +401,10 @@ Module contents ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have the same meaning as they do in :func:`dataclass`. + If ``module`` is defined, the ``__module__`` attribute + of the dataclass is set to that value. + By default, it is set to the module name of the caller. + This function is not strictly required, because any Python mechanism for creating a new class with ``__annotations__`` can then apply the :func:`dataclass` function to convert that class to diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 78a126f051e2e76..24f3779ebb8ec8e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1391,7 +1391,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False): + weakref_slot=False, module=None): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1455,6 +1455,19 @@ def exec_body_callback(ns): # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For pickling to work, the __module__ variable needs to be set to the frame + # where the dataclass is created. + if module is None: + try: + module = sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + cls.__module__ = module + # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 5486b2ef3f47e51..76bed0c33146734 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3606,6 +3606,15 @@ def test_text_annotations(self): 'return': type(None)}) +ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) +ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', + [('x', int)], + module='test.test_dataclasses') +WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) +WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', + [('x', int)], + module='custom') + class TestMakeDataclass(unittest.TestCase): def test_simple(self): C = make_dataclass('C', @@ -3715,6 +3724,36 @@ def test_no_types(self): 'y': int, 'z': 'typing.Any'}) + def test_module_attr(self): + self.assertEqual(ByMakeDataClass.__module__, __name__) + self.assertEqual(ByMakeDataClass(1).__module__, __name__) + self.assertEqual(WrongModuleMakeDataclass.__module__, "custom") + Nested = make_dataclass('Nested', []) + self.assertEqual(Nested.__module__, __name__) + self.assertEqual(Nested().__module__, __name__) + + def test_pickle_support(self): + for klass in [ByMakeDataClass, ManualModuleMakeDataClass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(klass, proto)), + klass, + ) + self.assertEqual( + pickle.loads(pickle.dumps(klass(1), proto)), + klass(1), + ) + + def test_cannot_be_pickled(self): + for klass in [WrongNameMakeDataclass, WrongModuleMakeDataclass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass, proto) + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass(1), proto) + def test_invalid_type_specification(self): for bad_field in [(), (1, 2, 3, 4), diff --git a/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst new file mode 100644 index 000000000000000..feba433f5bee899 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst @@ -0,0 +1,2 @@ +Add ``module`` argument to :func:`dataclasses.make_dataclass` and make +classes produced by it pickleable. From 767d3a8f6f2f94daa15ad3759d0ecdf4c009b7ab Mon Sep 17 00:00:00 2001 From: "Ilya V. Schurov" Date: Sat, 11 Mar 2023 01:27:41 +0100 Subject: [PATCH 30/35] gh-88071: Update docstrings of dataclass' astuple and asdict (#101806) Update dataclasses.astuple and dataclasses.asdict docstrings to reflect that they deep copy objects in the field values. --- Lib/dataclasses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 24f3779ebb8ec8e..f4617b1dbdac669 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1281,7 +1281,7 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1353,7 +1353,7 @@ class C: If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): From 08b67fb34f4519be1b0bb4673643a2c761c7ae92 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 10 Mar 2023 17:01:16 -0800 Subject: [PATCH 31/35] GH-90997: Shrink the LOAD_GLOBAL caches (#102569) --- Doc/library/dis.rst | 6 +- Include/internal/pycore_code.h | 2 +- Include/internal/pycore_opcode.h | 2 +- Lib/importlib/_bootstrap_external.py | 13 +- Lib/opcode.py | 2 +- Lib/test/test_dis.py | 292 +++++++++--------- ...3-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst | 2 + Python/bytecodes.c | 6 +- Python/generated_cases.c.h | 14 +- Python/opcode_metadata.h | 8 +- Python/specialize.c | 12 +- 11 files changed, 187 insertions(+), 172 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a5bc5e7fb6ea716..f4f47b3bf4846db 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -57,9 +57,9 @@ the following command can be used to display the disassembly of 2 0 RESUME 0 3 2 LOAD_GLOBAL 1 (NULL + len) - 14 LOAD_FAST 0 (alist) - 16 CALL 1 - 26 RETURN_VALUE + 12 LOAD_FAST 0 (alist) + 14 CALL 1 + 24 RETURN_VALUE (The "2" is a line number). diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 10f1e320a12ff43..6bd212dd42c6f27 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -20,7 +20,7 @@ extern "C" { typedef struct { uint16_t counter; uint16_t index; - uint16_t module_keys_version[2]; + uint16_t module_keys_version; uint16_t builtin_keys_version; } _PyLoadGlobalCache; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index f9ab95ca4bb9d34..da94fc81dcbe3e7 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -48,7 +48,7 @@ const uint8_t _PyOpcode_Caches[256] = { [STORE_ATTR] = 4, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, - [LOAD_GLOBAL] = 5, + [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, [SEND] = 1, [COMPARE_AND_BRANCH] = 1, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 954401cfa85ed3d..a01a0955182de5c 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -431,12 +431,17 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) # Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a5 3518 (Add RETURN_CONST instruction) -# Python 3.12a5 3519 (Modify SEND instruction) -# Python 3.12a5 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a6 3518 (Add RETURN_CONST instruction) +# Python 3.12a6 3519 (Modify SEND instruction) +# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) # Python 3.13 will start with 3550 +# Please don't copy-paste the same pre-release tag for new entries above!!! +# You should always use the *upcoming* tag. For example, if 3.12a6 came out +# a week ago, I should put "Python 3.12a7" next to my new magic number. + # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). @@ -446,7 +451,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3520).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3521).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 809d24e51676bde..23529d87a09ef94 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -390,7 +390,7 @@ def pseudo_op(name, op, real_ops): "LOAD_GLOBAL": { "counter": 1, "index": 1, - "module_keys_version": 2, + "module_keys_version": 1, "builtin_keys_version": 1, }, "BINARY_OP": { diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 9086824dd6f40cd..b77e3b06d0c1f1e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -138,10 +138,10 @@ def bug708901(): %3d CALL 2 GET_ITER - >> FOR_ITER 2 (to 38) + >> FOR_ITER 2 (to 36) STORE_FAST 0 (res) -%3d JUMP_BACKWARD 4 (to 30) +%3d JUMP_BACKWARD 4 (to 28) %3d >> END_FOR RETURN_CONST 0 (None) @@ -354,7 +354,7 @@ def bug42562(): %3d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to 82) + POP_JUMP_IF_FALSE 23 (to 80) STORE_FAST 0 (e) %3d LOAD_FAST 0 (e) @@ -730,14 +730,14 @@ def loop_test(): LOAD_CONST 2 (3) BINARY_OP 5 (*) GET_ITER - >> FOR_ITER_LIST 15 (to 50) + >> FOR_ITER_LIST 14 (to 48) STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (NULL + load_test) LOAD_FAST 0 (i) CALL_PY_WITH_DEFAULTS 1 POP_TOP - JUMP_BACKWARD 17 (to 16) + JUMP_BACKWARD 16 (to 16) %3d >> END_FOR RETURN_CONST 0 (None) @@ -1189,8 +1189,8 @@ def test_show_caches(self): caches = list(self.get_cached_values(quickened, adaptive)) for cache in caches: self.assertRegex(cache, pattern) - total_caches = 23 - empty_caches = 8 + total_caches = 22 + empty_caches = 7 self.assertEqual(caches.count(""), empty_caches) self.assertEqual(len(caches), total_caches) @@ -1483,17 +1483,17 @@ def _prepare_test_cases(): Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=20, starts_line=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=34, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=58, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=30, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=32, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=56, starts_line=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_f = [ @@ -1511,145 +1511,145 @@ def _prepare_test_cases(): Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=22, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=24, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=26, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=58, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=56, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, starts_line=3, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=4, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=40, starts_line=None, is_jump_target=False, positions=None) + Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=38, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, starts_line=1, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='NULL + range', offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=88, argrepr='to 88', offset=28, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=34, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=60, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=62, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=64, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=72, argrepr='to 72', offset=68, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=28, argrepr='to 28', offset=70, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=72, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=84, argrepr='to 84', offset=80, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=28, argval=28, argrepr='to 28', offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=84, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=14, argval=116, argrepr='to 116', offset=86, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=88, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=90, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=104, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=114, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=116, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=33, argval=186, argrepr='to 186', offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=120, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=132, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=134, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=146, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=154, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=156, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=158, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=160, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=168, argrepr='to 168', offset=164, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=116, argrepr='to 116', offset=166, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=168, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=172, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=180, argrepr='to 180', offset=176, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=16, argval=212, argrepr='to 212', offset=178, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=180, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=186, argrepr='to 186', offset=182, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=33, argval=120, argrepr='to 120', offset=184, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=186, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=198, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=210, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=212, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=214, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=216, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=218, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=222, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=224, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=228, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=230, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=242, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=244, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=256, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=258, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=272, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=274, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=286, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=288, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=302, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=304, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=310, argrepr='to 310', offset=306, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=310, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=314, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=316, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=23, argval=274, argrepr='to 274', offset=318, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=320, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=322, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=328, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=340, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=16, argval=376, argrepr='to 376', offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=344, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=346, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=370, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=372, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=51, argval=274, argrepr='to 274', offset=374, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=376, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=386, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=398, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=400, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=410, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=412, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=414, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=416, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=418, starts_line=None, is_jump_target=False, positions=None) + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=93, arg=27, argval=84, argrepr='to 84', offset=26, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=30, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=32, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=56, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=58, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=68, argrepr='to 68', offset=64, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=26, argrepr='to 26', offset=66, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=68, starts_line=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=70, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=72, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=80, argrepr='to 80', offset=76, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=27, argval=26, argrepr='to 26', offset=78, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, starts_line=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=13, argval=110, argrepr='to 110', offset=82, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, starts_line=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=108, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=110, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=32, argval=178, argrepr='to 178', offset=112, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=114, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=126, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=136, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=138, starts_line=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=140, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=142, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=146, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=148, starts_line=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=150, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=152, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=160, argrepr='to 160', offset=156, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=25, argval=110, argrepr='to 110', offset=158, starts_line=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=160, starts_line=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=162, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=164, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=172, argrepr='to 172', offset=168, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=15, argval=202, argrepr='to 202', offset=170, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=172, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=178, argrepr='to 178', offset=174, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=32, argval=114, argrepr='to 114', offset=176, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=178, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=188, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=190, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=202, starts_line=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=204, starts_line=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=206, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=208, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=214, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=216, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=218, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=220, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=230, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=232, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=242, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=244, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=246, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=248, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=250, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=260, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=262, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=272, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=274, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=284, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=286, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=288, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=290, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=296, argrepr='to 296', offset=292, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=294, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=296, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=300, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=302, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=262, argrepr='to 262', offset=304, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=306, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=310, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=314, starts_line=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=358, argrepr='to 358', offset=326, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=328, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=330, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=340, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=342, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=352, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=354, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=48, argval=262, argrepr='to 262', offset=356, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=358, starts_line=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=362, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=364, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=366, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=368, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=378, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=390, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=392, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=394, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=396, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=398, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst new file mode 100644 index 000000000000000..723a4b9fa777d64 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst @@ -0,0 +1,2 @@ +Shrink the number of inline :opcode:`CACHE` entries used by +:opcode:`LOAD_GLOBAL`. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 45d507266743211..dc2a20f22b6038f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1078,7 +1078,7 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/2, unused/1 -- null if (oparg & 1), v)) { + inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- null if (oparg & 1), v)) { #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1133,7 +1133,7 @@ dummy_func( null = NULL; } - inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/2, unused/1 -- null if (oparg & 1), res)) { + inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/1, unused/1 -- null if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1147,7 +1147,7 @@ dummy_func( null = NULL; } - inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/2, bltn_version/1 -- null if (oparg & 1), res)) { + inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/1, bltn_version/1 -- null if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 51357cda4c8949c..3839aeecb912a10 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1349,7 +1349,7 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 5, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; #if ENABLE_SPECIALIZATION @@ -1408,7 +1408,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } @@ -1416,7 +1416,7 @@ PyObject *null = NULL; PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); - uint32_t version = read_u32(&next_instr[2].cache); + uint16_t version = read_u16(&next_instr[2].cache); assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1432,7 +1432,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } @@ -1440,8 +1440,8 @@ PyObject *null = NULL; PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); - uint32_t mod_version = read_u32(&next_instr[2].cache); - uint16_t bltn_version = read_u16(&next_instr[4].cache); + uint16_t mod_version = read_u16(&next_instr[2].cache); + uint16_t bltn_version = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); @@ -1460,7 +1460,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 67cb0088c3b7893..93f3c76c5b82406 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -708,7 +708,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { #endif enum Direction { DIR_NONE, DIR_READ, DIR_WRITE }; -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC0000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { enum Direction dir_op1; enum Direction dir_op2; @@ -790,9 +790,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, - [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, - [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, + [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [DELETE_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_CELL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_DEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/specialize.c b/Python/specialize.c index 0a7af8991eec83c..719bd5bda329ff3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1148,8 +1148,12 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (keys_version != (uint16_t)keys_version) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } cache->index = (uint16_t)index; - write_u32(cache->module_keys_version, keys_version); + cache->module_keys_version = (uint16_t)keys_version; instr->op.code = LOAD_GLOBAL_MODULE; goto success; } @@ -1177,6 +1181,10 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (globals_version != (uint16_t)globals_version) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( interp, builtin_keys); if (builtins_version == 0) { @@ -1188,7 +1196,7 @@ _Py_Specialize_LoadGlobal( goto fail; } cache->index = (uint16_t)index; - write_u32(cache->module_keys_version, globals_version); + cache->module_keys_version = (uint16_t)globals_version; cache->builtin_keys_version = (uint16_t)builtins_version; instr->op.code = LOAD_GLOBAL_BUILTIN; goto success; From 5ffdaf748d98da6065158534720f1996a45a0072 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 11 Mar 2023 01:20:20 +0000 Subject: [PATCH 32/35] gh-102433: Add tests for how classes with properties interact with `isinstance()` checks on `typing.runtime_checkable` protocols (#102449) Co-authored-by: Carl Meyer --- Lib/test/test_typing.py | 88 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0483ca3aa42f94b..c17be6cd0bbc4a5 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2530,6 +2530,94 @@ def meth(x): ... with self.assertRaises(TypeError): isinstance(C(), BadPG) + def test_protocols_isinstance_properties_and_descriptors(self): + class C: + @property + def attr(self): + return 42 + + class CustomDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + class D: + attr = CustomDescriptor() + + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... + + class Empty: ... + + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int + + @runtime_checkable + class PG(Protocol[T]): + @property + def attr(self): ... + + @runtime_checkable + class PG1(Protocol[T]): + attr: T + + for protocol_class in P, P1, PG, PG1: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) + + with self.subTest(klass="Empty", protocol_class=protocol_class.__name__): + self.assertNotIsInstance(Empty(), protocol_class) + + class BadP(Protocol): + @property + def attr(self): ... + + class BadP1(Protocol): + attr: int + + class BadPG(Protocol[T]): + @property + def attr(self): ... + + class BadPG1(Protocol[T]): + attr: T + + for obj in PG[T], PG[C], PG1[T], PG1[C], BadP, BadP1, BadPG, BadPG1: + for klass in C, D, E, F, Empty: + with self.subTest(klass=klass.__name__, obj=obj): + with self.assertRaises(TypeError): + isinstance(klass(), obj) + + def test_protocols_isinstance_not_fooled_by_custom_dir(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class CustomDirWithX: + x = 10 + def __dir__(self): + return [] + + class CustomDirWithoutX: + def __dir__(self): + return ["x"] + + self.assertIsInstance(CustomDirWithX(), HasX) + self.assertNotIsInstance(CustomDirWithoutX(), HasX) + def test_protocols_isinstance_py36(self): class APoint: def __init__(self, x, y, label): From aa0a73d1bc53dcb6348a869df1e775138991e561 Mon Sep 17 00:00:00 2001 From: wangxiang-hz <34048878+wangxiang-hz@users.noreply.github.com> Date: Sat, 11 Mar 2023 19:11:37 +0800 Subject: [PATCH 33/35] gh-102213: Optimize the performance of `__getattr__` (GH-102248) When __getattr__ is defined, python with try to find an attribute using _PyObject_GenericGetAttrWithDict find nothing is reasonable so we don't need an exception, it will hurt performance. --- Include/internal/pycore_object.h | 1 + .../2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst | 1 + Objects/object.c | 6 ++++++ Objects/typeobject.c | 9 ++++++--- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e15685f174ebcfe..318e6f3371c0c35 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -370,6 +370,7 @@ extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); +extern PyObject* _PyObject_GenericTryGetAttr(PyObject *, PyObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst new file mode 100644 index 000000000000000..997bef226e713f3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst @@ -0,0 +1 @@ +Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Objects/object.c b/Objects/object.c index 38da4d497a96e72..dff5e2afa16ab84 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1405,6 +1405,12 @@ PyObject_GenericGetAttr(PyObject *obj, PyObject *name) return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 0); } +PyObject * +_PyObject_GenericTryGetAttr(PyObject *obj, PyObject *name) +{ + return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 1); +} + int _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, PyObject *value, PyObject *dict) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f486b83fd69e648..f0654c239f66358 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8247,14 +8247,17 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) - res = PyObject_GenericGetAttr(self, name); + /* finding nothing is reasonable when __getattr__ is defined */ + res = _PyObject_GenericTryGetAttr(self, name); else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); } - if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } res = call_attribute(self, getattr, name); } Py_DECREF(getattr); From ced13c96a4eb9391a9d27e3e13218f70c579670f Mon Sep 17 00:00:00 2001 From: Thomas Krennwallner Date: Sat, 11 Mar 2023 08:19:40 -0500 Subject: [PATCH 34/35] gh-79940: add introspection API for asynchronous generators to `inspect` module (#11590) --- Doc/library/inspect.rst | 28 ++++- Doc/whatsnew/3.12.rst | 4 + Lib/inspect.py | 50 ++++++++ Lib/test/test_inspect.py | 107 ++++++++++++++++++ ...3-02-26-17-29-57.gh-issue-79940.SAfmAy.rst | 2 + Objects/genobject.c | 10 ++ 6 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 789e9839d22f71d..ccf240193d36a9e 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1440,8 +1440,8 @@ code execution:: pass -Current State of Generators and Coroutines ------------------------------------------- +Current State of Generators, Coroutines, and Asynchronous Generators +-------------------------------------------------------------------- When implementing coroutine schedulers and for other advanced uses of generators, it is useful to determine whether a generator is currently @@ -1476,6 +1476,22 @@ generator to be determined easily. .. versionadded:: 3.5 +.. function:: getasyncgenstate(agen) + + Get current state of an asynchronous generator object. The function is + intended to be used with asynchronous iterator objects created by + :keyword:`async def` functions which use the :keyword:`yield` statement, + but will accept any asynchronous generator-like object that has + ``ag_running`` and ``ag_frame`` attributes. + + Possible states are: + * AGEN_CREATED: Waiting to start execution. + * AGEN_RUNNING: Currently being executed by the interpreter. + * AGEN_SUSPENDED: Currently suspended at a yield expression. + * AGEN_CLOSED: Execution has completed. + + .. versionadded:: 3.12 + The current internal state of the generator can also be queried. This is mostly useful for testing purposes, to ensure that internal state is being updated as expected: @@ -1507,6 +1523,14 @@ updated as expected: .. versionadded:: 3.5 +.. function:: getasyncgenlocals(agen) + + This function is analogous to :func:`~inspect.getgeneratorlocals`, but + works for asynchronous generator objects created by :keyword:`async def` + functions which use the :keyword:`yield` statement. + + .. versionadded:: 3.12 + .. _inspect-module-co-flags: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 48b7aab0595ebb2..9f33dbc808ddc02 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -244,6 +244,10 @@ inspect a :term:`coroutine` for use with :func:`iscoroutinefunction`. (Contributed Carlton Gibson in :gh:`99247`.) +* Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` + for determining the current state of asynchronous generators. + (Contributed by Thomas Krennwallner in :issue:`35759`.) + pathlib ------- diff --git a/Lib/inspect.py b/Lib/inspect.py index edc23b0ffa9201f..0eceaaf9a24f5d6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -34,6 +34,10 @@ 'Yury Selivanov ') __all__ = [ + "AGEN_CLOSED", + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", "ArgInfo", "Arguments", "Attribute", @@ -77,6 +81,8 @@ "getabsfile", "getargs", "getargvalues", + "getasyncgenlocals", + "getasyncgenstate", "getattr_static", "getblock", "getcallargs", @@ -1935,6 +1941,50 @@ def getcoroutinelocals(coroutine): return {} +# ----------------------------------- asynchronous generator introspection + +AGEN_CREATED = 'AGEN_CREATED' +AGEN_RUNNING = 'AGEN_RUNNING' +AGEN_SUSPENDED = 'AGEN_SUSPENDED' +AGEN_CLOSED = 'AGEN_CLOSED' + + +def getasyncgenstate(agen): + """Get current state of an asynchronous generator object. + + Possible states are: + AGEN_CREATED: Waiting to start execution. + AGEN_RUNNING: Currently being executed by the interpreter. + AGEN_SUSPENDED: Currently suspended at a yield expression. + AGEN_CLOSED: Execution has completed. + """ + if agen.ag_running: + return AGEN_RUNNING + if agen.ag_suspended: + return AGEN_SUSPENDED + if agen.ag_frame is None: + return AGEN_CLOSED + return AGEN_CREATED + + +def getasyncgenlocals(agen): + """ + Get the mapping of asynchronous generator local variables to their current + values. + + A dict is returned, with the keys the local variable names and values the + bound values.""" + + if not isasyncgen(agen): + raise TypeError(f"{agen!r} is not a Python async generator") + + frame = getattr(agen, "ag_frame", None) + if frame is not None: + return agen.ag_frame.f_locals + else: + return {} + + ############################################################################### ### Function Signature Object (PEP 362) ############################################################################### diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 02f8378d0413eac..410a2e5b5468f02 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1,3 +1,4 @@ +import asyncio import builtins import collections import datetime @@ -65,6 +66,10 @@ def revise(filename, *args): git = mod.StupidGit() +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -2321,6 +2326,108 @@ async def func(a=None): {'a': None, 'gencoro': gencoro, 'b': 'spam'}) +class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + async def number_asyncgen(): + for number in range(5): + yield number + self.asyncgen = number_asyncgen() + + async def asyncTearDown(self): + await self.asyncgen.aclose() + + def _asyncgenstate(self): + return inspect.getasyncgenstate(self.asyncgen) + + def test_created(self): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CREATED) + + async def test_suspended(self): + value = await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + self.assertEqual(value, 0) + + async def test_closed_after_exhaustion(self): + countdown = 7 + with self.assertRaises(StopAsyncIteration): + while countdown := countdown - 1: + await anext(self.asyncgen) + self.assertEqual(countdown, 1) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + async def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + await self.asyncgen.athrow(RuntimeError) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + async def test_running(self): + async def running_check_asyncgen(): + for number in range(5): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + yield number + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + self.asyncgen = running_check_asyncgen() + # Running up to the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + # Running after the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + + def test_easy_debugging(self): + # repr() and str() of a asyncgen state should contain the state name + names = 'AGEN_CREATED AGEN_RUNNING AGEN_SUSPENDED AGEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + async def test_getasyncgenlocals(self): + async def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + with self.assertRaises(StopAsyncIteration): + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), {}) + + async def test_getasyncgenlocals_empty(self): + async def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getasyncgenlocals(one), {}) + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + with self.assertRaises(StopAsyncIteration): + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + + def test_getasyncgenlocals_error(self): + self.assertRaises(TypeError, inspect.getasyncgenlocals, 1) + self.assertRaises(TypeError, inspect.getasyncgenlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getasyncgenlocals, set) + self.assertRaises(TypeError, inspect.getasyncgenlocals, (2,3)) + + class MySignature(inspect.Signature): # Top-level to make it picklable; # used in test_signature_object_pickle diff --git a/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst new file mode 100644 index 000000000000000..31b8ead8433279b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst @@ -0,0 +1,2 @@ +Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals`. +Patch by Thomas Krennwallner. diff --git a/Objects/genobject.c b/Objects/genobject.c index 61463774310f884..6316fa9865fe651 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1520,6 +1520,15 @@ ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) return _gen_getcode(gen, "ag_code"); } +static PyObject * +ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +{ + if (ag->ag_frame_state == FRAME_SUSPENDED) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyGetSetDef async_gen_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the async generator")}, @@ -1529,6 +1538,7 @@ static PyGetSetDef async_gen_getsetlist[] = { PyDoc_STR("object being awaited on, or None")}, {"ag_frame", (getter)ag_getframe, NULL, NULL}, {"ag_code", (getter)ag_getcode, NULL, NULL}, + {"ag_suspended", (getter)ag_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; From 534660f1680955c7a6a47d5c6bd9649704b74a87 Mon Sep 17 00:00:00 2001 From: Thomas Krennwallner Date: Sat, 11 Mar 2023 13:36:49 -0500 Subject: [PATCH 35/35] gh-79940: skip `TestGetAsyncGenState` on wasm as it requires working sockets (GH-102605) Skip `TestGetAsyncGenState` and restoring of the default event loop policy in `test_inspect` if platform lacks working socket support. Fixes #11590 Automerge-Triggered-By: GH:kumaraditya303 --- Lib/test/test_inspect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 410a2e5b5468f02..803b259d961f540 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -67,7 +67,8 @@ def revise(filename, *args): def tearDownModule(): - asyncio.set_event_loop_policy(None) + if support.has_socket_support: + asyncio.set_event_loop_policy(None) def signatures_with_lexicographic_keyword_only_parameters(): @@ -2326,6 +2327,7 @@ async def func(a=None): {'a': None, 'gencoro': gencoro, 'b': 'spam'}) +@support.requires_working_socket() class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): def setUp(self):