From 8ff13dee440f8fda3f45e8c96fbe12811cc787a0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 20 Nov 2022 09:52:55 +0200 Subject: [PATCH 1/3] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data Previously *consumed was not set in this case. --- Lib/test/test_capi/test_unicode.py | 47 +++++++++++++++++++ ...2-11-20-09-52-50.gh-issue-99612.eBHksg.rst | 2 + Modules/_testcapi/unicode.c | 37 +++++++++++++++ Objects/unicodeobject.c | 3 ++ 4 files changed, 89 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 07b77d3e04bbe0..f7cb0974a41c4c 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -11,6 +11,53 @@ class CAPITest(unittest.TestCase): + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + from _testcapi import unicode_decodeutf8 as decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + from _testcapi import unicode_decodeutf8stateful as decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + # Test PyUnicode_FromFormat() def test_from_format(self): import_helper.import_module('ctypes') diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst new file mode 100644 index 00000000000000..40e3c8db5403c7 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst @@ -0,0 +1,2 @@ +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index d5c4a9e5b95ec6..aa46d72ba3d595 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include "parts.h" static struct PyModuleDef *_testcapimodule = NULL; // set at initialization @@ -223,6 +224,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, utf8_len); } +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + static PyObject * unicode_count(PyObject *self, PyObject *args) { @@ -716,6 +751,8 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, {"unicode_count", unicode_count, METH_VARARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b1acfc71379cd5..34e08f2ac1abc0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4530,6 +4530,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, } s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); if (s == end) { + if (consumed) { + *consumed = size; + } return u; } From b99be83be8265de6a5aef221d57879dc8c98c61d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Nov 2022 14:06:59 +0200 Subject: [PATCH 2/3] Move tests to test_capi.test_codecs. --- Lib/test/test_capi/test_codecs.py | 61 ++++++++++++++++++++++++++++++ Lib/test/test_capi/test_unicode.py | 47 ----------------------- 2 files changed, 61 insertions(+), 47 deletions(-) create mode 100644 Lib/test/test_capi/test_codecs.py diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py new file mode 100644 index 00000000000000..03d8be01b49ebd --- /dev/null +++ b/Lib/test/test_capi/test_codecs.py @@ -0,0 +1,61 @@ +import unittest +from test import support + +try: + import _testcapi +except ImportError: + _testcapi = None + + +class CAPITest(unittest.TestCase): + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + from _testcapi import unicode_decodeutf8 as decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + from _testcapi import unicode_decodeutf8stateful as decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index f7cb0974a41c4c..07b77d3e04bbe0 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -11,53 +11,6 @@ class CAPITest(unittest.TestCase): - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_decodeutf8(self): - """Test PyUnicode_DecodeUTF8()""" - from _testcapi import unicode_decodeutf8 as decodeutf8 - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-8') - self.assertEqual(decodeutf8(b), s) - self.assertEqual(decodeutf8(b, 'strict'), s) - - self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') - self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') - self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') - self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') - self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') - self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') - - self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') - # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and - # negative size. - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_decodeutf8stateful(self): - """Test PyUnicode_DecodeUTF8Stateful()""" - from _testcapi import unicode_decodeutf8stateful as decodeutf8stateful - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-8') - self.assertEqual(decodeutf8stateful(b), (s, len(b))) - self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) - - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') - self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) - self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') - self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) - - self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') - # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and - # negative size. - # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of - # "consumed". - # Test PyUnicode_FromFormat() def test_from_format(self): import_helper.import_module('ctypes') From fd8c21d9366411a26923696bc8ff66f5bbc8dc7a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 30 Nov 2022 19:04:48 +0200 Subject: [PATCH 3/3] Address review comments. --- Lib/test/test_capi/test_codecs.py | 15 ++++----------- Modules/_testcapi/unicode.c | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 03d8be01b49ebd..e46726192aa05b 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -1,19 +1,14 @@ import unittest -from test import support +from test.support import import_helper -try: - import _testcapi -except ImportError: - _testcapi = None +_testcapi = import_helper.import_module('_testcapi') class CAPITest(unittest.TestCase): - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_decodeutf8(self): """Test PyUnicode_DecodeUTF8()""" - from _testcapi import unicode_decodeutf8 as decodeutf8 + decodeutf8 = _testcapi.unicode_decodeutf8 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -31,11 +26,9 @@ def test_decodeutf8(self): # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and # negative size. - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_decodeutf8stateful(self): """Test PyUnicode_DecodeUTF8Stateful()""" - from _testcapi import unicode_decodeutf8stateful as decodeutf8stateful + decodeutf8stateful = _testcapi.unicode_decodeutf8stateful for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 332a6eaca2a48e..2d23993ce420b3 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -260,7 +260,7 @@ unicode_decodeutf8stateful(PyObject *self, PyObject *args) const char *data; Py_ssize_t size; const char *errors = NULL; - Py_ssize_t consumed; + Py_ssize_t consumed = 123456789; PyObject *result; if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors))