From ca76251558cef8e6b5731a1f9f3e86d32e2ff3ac Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 2 Oct 2022 15:43:42 -0700 Subject: [PATCH 01/35] try 1 --- Include/internal/pycore_global_strings.h | 2 + Objects/typeobject.c | 106 ++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 1523eef73931c9..f79e6ad2d53827 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -74,6 +74,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) + STRUCT_FOR_ID(__buffer__) STRUCT_FOR_ID(__build_class__) STRUCT_FOR_ID(__builtins__) STRUCT_FOR_ID(__bytes__) @@ -172,6 +173,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__rdivmod__) STRUCT_FOR_ID(__reduce__) STRUCT_FOR_ID(__reduce_ex__) + STRUCT_FOR_ID(__release_buffer__) STRUCT_FOR_ID(__repr__) STRUCT_FOR_ID(__reversed__) STRUCT_FOR_ID(__rfloordiv__) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5aa5cbbd54022e..fc3e0f42d884c4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7551,6 +7551,46 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) Py_RETURN_NONE; } +static PyObject * +wrap_buffer(PyObject *self, PyObject *args, void *wrapped) +{ + getbufferproc func = (getbufferproc)wrapped; + Py_buffer view; + int flags = 0; + + if (!check_num_args(args, 1)) { + return NULL; + } + if (!PyArg_ParseTuple(args, "i", &flags)) { + return NULL; + } + if ((*func)(self, &view, flags) < 0) { + return NULL; + } + return PyMemoryView_FromBuffer(&view); +} + +static PyObject * +wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) +{ + releasebufferproc func = (releasebufferproc)wrapped; + PyMemoryViewObject *mview; + + if (!check_num_args(args, 1)) { + return NULL; + } + if (!PyArg_ParseTuple(args, "O!", &PyMemoryView_Type, &mview)) { + return NULL; + } + if (mview->view.obj != self) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer is not this object"); + return NULL; + } + (*func)(self, &mview->view); + Py_RETURN_NONE; +} + static PyObject * wrap_init(PyObject *self, PyObject *args, void *wrapped, PyObject *kwds) { @@ -8383,6 +8423,54 @@ slot_tp_finalize(PyObject *self) PyErr_Restore(error_type, error_value, error_traceback); } +static int +slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) +{ + PyObject *flags_obj = PyLong_FromLong(flags); + if (flags_obj == NULL) { + return -1; + } + PyObject *stack[2] = {self, flags_obj}; + PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); + if (ret == NULL) { + goto fail; + } + if (!PyMemoryView_Check(ret)) { + PyErr_Format(PyExc_TypeError, + "__buffer__ returned non-memoryview object"); + goto fail; + } + *buffer = ((PyMemoryViewObject *)ret)->view; + Py_DECREF(ret); + Py_DECREF(flags_obj); + return 0; + +fail: + Py_XDECREF(ret); + Py_DECREF(flags_obj); + return -1; +} + +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + PyObject *mv = PyMemoryView_FromBuffer(buffer); + if (mv == NULL) { + PyErr_WriteUnraisable(self); + return; + } + PyObject *stack[2] = {self, mv}; + PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); + Py_DECREF(mv); + if (ret == NULL) { + PyErr_WriteUnraisable(self); + } + else { + Py_DECREF(ret); + } +} + + static PyObject * slot_am_await(PyObject *self) { @@ -8452,6 +8540,7 @@ typedef struct wrapperbase slotdef; #undef TPSLOT #undef FLSLOT +#undef BUFSLOT #undef AMSLOT #undef ETSLOT #undef SQSLOT @@ -8471,6 +8560,8 @@ typedef struct wrapperbase slotdef; #define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ {#NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ PyDoc_STR(DOC), .name_strobj = &_Py_ID(NAME) } +#define BUFSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ + ETSLOT(NAME, as_buffer.SLOT, FUNCTION, WRAPPER, DOC) #define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC) #define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ @@ -8552,6 +8643,13 @@ static slotdef slotdefs[] = { "Create and return new object. See help(type) for accurate signature."), TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, + "__buffer__($self, flags, /)\n--\n\n" + "Return a buffer object that exposes the underlying memory of the object."), + BUFSLOT(__release_buffer__, bf_releasebuffer, slot_bf_releasebuffer, wrap_releasebuffer, + "__release_buffer__($self, /)\n--\n\n" + "Release the buffer object that exposes the underlying memory of the object."), + AMSLOT(__await__, am_await, slot_am_await, wrap_unaryfunc, "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."), AMSLOT(__aiter__, am_aiter, slot_am_aiter, wrap_unaryfunc, @@ -8698,8 +8796,12 @@ slotptr(PyTypeObject *type, int ioffset) /* Note: this depends on the order of the members of PyHeapTypeObject! */ assert(offset >= 0); - assert((size_t)offset < offsetof(PyHeapTypeObject, as_buffer)); - if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { + assert((size_t)offset < offsetof(PyHeapTypeObject, ht_name)); + if ((size_t)offset >= offsetof(PyHeapTypeObject, as_buffer)) { + ptr = (char *)type->tp_as_buffer; + offset -= offsetof(PyHeapTypeObject, as_buffer); + } + else if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { ptr = (char *)type->tp_as_sequence; offset -= offsetof(PyHeapTypeObject, as_sequence); } From 901b459cd92e224afddc5291e3670cd0efdd3511 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 3 Oct 2022 11:50:52 -0700 Subject: [PATCH 02/35] progress --- Include/internal/pycore_global_strings.h | 2 +- Include/internal/pycore_memoryobject.h | 17 +++++++ .../internal/pycore_runtime_init_generated.h | 14 +++++ Lib/test/test_buffer.py | 51 +++++++++++++++++++ Objects/memoryobject.c | 21 +++++--- Objects/typeobject.c | 15 +++--- 6 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 Include/internal/pycore_memoryobject.h diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f79e6ad2d53827..45068c5aabe960 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -173,7 +173,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__rdivmod__) STRUCT_FOR_ID(__reduce__) STRUCT_FOR_ID(__reduce_ex__) - STRUCT_FOR_ID(__release_buffer__) + STRUCT_FOR_ID(__release_buffer__) STRUCT_FOR_ID(__repr__) STRUCT_FOR_ID(__reversed__) STRUCT_FOR_ID(__rfloordiv__) diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h new file mode 100644 index 00000000000000..acc12c9275172c --- /dev/null +++ b/Include/internal/pycore_memoryobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_MEMORYOBJECT_H +#define Py_INTERNAL_MEMORYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyObject * +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_MEMORYOBJECT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 32ff57b731e1a0..662ff2b3151b84 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -583,6 +583,7 @@ extern "C" { INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ + INIT_ID(__buffer__), \ INIT_ID(__build_class__), \ INIT_ID(__builtins__), \ INIT_ID(__bytes__), \ @@ -681,6 +682,7 @@ extern "C" { INIT_ID(__rdivmod__), \ INIT_ID(__reduce__), \ INIT_ID(__reduce_ex__), \ + INIT_ID(__release_buffer__), \ INIT_ID(__repr__), \ INIT_ID(__reversed__), \ INIT_ID(__rfloordiv__), \ @@ -1472,6 +1474,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__bool__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__build_class__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__builtins__); @@ -1668,6 +1672,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce_ex__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__release_buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__repr__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reversed__); @@ -4871,6 +4877,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(__bool__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(__buffer__)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(__buffer__)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(__build_class__)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(__build_class__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); @@ -5263,6 +5273,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(__reduce_ex__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(__release_buffer__)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(__release_buffer__)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(__repr__)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(__repr__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 8ac3b7e7eb29d1..a1b624aef4605c 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4436,5 +4436,56 @@ def test_pybuffer_size_from_format(self): struct.calcsize(format)) +class TestPythonBufferProtocol(unittest.TestCase): + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + def __release_buffer__(self, buffer): + assert self is buffer.obj + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index c5ab0bf2dedbe4..db832aa5c9b22a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -85,7 +85,7 @@ mbuf_alloc(void) } static PyObject * -_PyManagedBuffer_FromObject(PyObject *base) +_PyManagedBuffer_FromObject(PyObject *base, int flags) { _PyManagedBufferObject *mbuf; @@ -93,7 +93,7 @@ _PyManagedBuffer_FromObject(PyObject *base) if (mbuf == NULL) return NULL; - if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL_RO) < 0) { + if (PyObject_GetBuffer(base, &mbuf->master, flags) < 0) { mbuf->master.obj = NULL; Py_DECREF(mbuf); return NULL; @@ -779,11 +779,12 @@ PyMemoryView_FromBuffer(const Py_buffer *info) return mv; } -/* Create a memoryview from an object that implements the buffer protocol. +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ PyObject * -PyMemoryView_FromObject(PyObject *v) +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -794,7 +795,7 @@ PyMemoryView_FromObject(PyObject *v) } else if (PyObject_CheckBuffer(v)) { PyObject *ret; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v, flags); if (mbuf == NULL) return NULL; ret = mbuf_add_view(mbuf, NULL); @@ -807,6 +808,14 @@ PyMemoryView_FromObject(PyObject *v) Py_TYPE(v)->tp_name); return NULL; } +/* Create a memoryview from an object that implements the buffer protocol. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +PyMemoryView_FromObject(PyObject *v) +{ + return PyMemoryView_FromObjectAndFlags(v, PyBUF_FULL_RO); +} /* Copy the format string from a base object that might vanish. */ static int @@ -853,7 +862,7 @@ memory_from_contiguous_copy(const Py_buffer *src, char order) if (bytes == NULL) return NULL; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes, PyBUF_FULL_RO); Py_DECREF(bytes); if (mbuf == NULL) return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fc3e0f42d884c4..48baac313151d9 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5,6 +5,7 @@ #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -7554,8 +7555,6 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) static PyObject * wrap_buffer(PyObject *self, PyObject *args, void *wrapped) { - getbufferproc func = (getbufferproc)wrapped; - Py_buffer view; int flags = 0; if (!check_num_args(args, 1)) { @@ -7564,10 +7563,7 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) if (!PyArg_ParseTuple(args, "i", &flags)) { return NULL; } - if ((*func)(self, &view, flags) < 0) { - return NULL; - } - return PyMemoryView_FromBuffer(&view); + return PyMemoryView_FromObjectAndFlags(self, flags); } static PyObject * @@ -8441,6 +8437,9 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) goto fail; } *buffer = ((PyMemoryViewObject *)ret)->view; + // TODO does this leak the existing ref in buffer->obj? + buffer->obj = self; + Py_INCREF(buffer->obj); Py_DECREF(ret); Py_DECREF(flags_obj); return 0; @@ -8459,6 +8458,9 @@ slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) PyErr_WriteUnraisable(self); return; } + if (buffer->obj != NULL) { + ((PyMemoryViewObject *)mv)->view.obj = Py_NewRef(buffer->obj); + } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); Py_DECREF(mv); @@ -8470,7 +8472,6 @@ slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) } } - static PyObject * slot_am_await(PyObject *self) { From c5407c7dfb30d6926569a21b00c0c0ccf5b2e89a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Oct 2022 10:37:50 -0700 Subject: [PATCH 03/35] this is better --- Lib/test/test_buffer.py | 2 +- Objects/typeobject.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index a1b624aef4605c..bd6bca1f48d0d5 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4478,7 +4478,7 @@ def __release_buffer__(self, buffer): with memoryview(wr) as mv: self.assertTrue(wr.held) self.assertEqual(mv.tobytes(), b"hello") - self.assertFalse(wr.held) + #self.assertFalse(wr.held) def test_call_builtins(self): ba = bytearray(b"hello") diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 48baac313151d9..24b260d2a3cb83 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8437,8 +8437,6 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) goto fail; } *buffer = ((PyMemoryViewObject *)ret)->view; - // TODO does this leak the existing ref in buffer->obj? - buffer->obj = self; Py_INCREF(buffer->obj); Py_DECREF(ret); Py_DECREF(flags_obj); From 86f2000a67fa1f990dbae3cc7400ea4fa042cec8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Oct 2022 16:59:28 -0700 Subject: [PATCH 04/35] this seems to work --- Include/internal/pycore_typeobject.h | 2 + Lib/_collections_abc.py | 32 ++++++++++- Lib/test/test_buffer.py | 3 +- Lib/test/test_collections.py | 18 ++++++- Objects/object.c | 2 + Objects/typeobject.c | 80 ++++++++++++++++++++++++++-- 6 files changed, 128 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 5e7aca1b9f5ae9..97309650c895c1 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -75,6 +75,8 @@ extern void _PyStaticType_Dealloc(PyTypeObject *type); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type; + #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c62233b81a5c95..3b0d0eeb1fa9d1 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", + "ByteString", "Buffer", "MutableBuffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -439,6 +439,36 @@ def __subclasshook__(cls, C): return NotImplemented +class Buffer(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + if cls is Buffer: + return _check_methods(C, "__buffer__") + return NotImplemented + + +class MutableBuffer(Buffer): + + __slots__ = () + + @abstractmethod + def __release_buffer__(self, buffer: memoryview, /) -> None: + pass + + @classmethod + def __subclasshook__(cls, C): + if cls is MutableBuffer: + return _check_methods(C, "__buffer__", "__release_buffer__") + return NotImplemented + + class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index bd6bca1f48d0d5..c46045ae96485e 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4470,7 +4470,6 @@ def __buffer__(self, flags): self.held = True return memoryview(self.ba) def __release_buffer__(self, buffer): - assert self is buffer.obj self.held = False wr = WhatToRelease() @@ -4478,7 +4477,7 @@ def __release_buffer__(self, buffer): with memoryview(wr) as mv: self.assertTrue(wr.held) self.assertEqual(mv.tobytes(), b"hello") - #self.assertFalse(wr.held) + self.assertFalse(wr.held) def test_call_builtins(self): ba = bytearray(b"hello") diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 59b3f2ec7bfcb6..a8ea91d973638f 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -25,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString +from collections.abc import ByteString, Buffer, MutableBuffer class TestUserObjects(unittest.TestCase): @@ -1941,6 +1941,22 @@ def test_ByteString(self): self.assertNotIsInstance(memoryview(b""), ByteString) self.assertFalse(issubclass(memoryview, ByteString)) + def test_Buffer(self): + for sample in [bytes, bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), Buffer) + self.assertTrue(issubclass(sample, Buffer)) + for sample in [str, list, tuple]: + self.assertNotIsInstance(sample(), Buffer) + self.assertFalse(issubclass(sample, Buffer)) + + def test_MutableBuffer(self): + for sample in [bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), MutableBuffer) + self.assertTrue(issubclass(sample, MutableBuffer)) + for sample in [bytes, str, list, tuple]: + self.assertNotIsInstance(sample(), MutableBuffer) + self.assertFalse(issubclass(sample, MutableBuffer)) + def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) diff --git a/Objects/object.c b/Objects/object.c index 26cdcef59dddbd..5edbe34025daa6 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -14,6 +14,7 @@ #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type +#include "pycore_typeobject.h" // _PyBufferWrapper_Type #include "pycore_unionobject.h" // _PyUnion_Type #include "pycore_interpreteridobject.h" // _PyInterpreterID_Type @@ -1930,6 +1931,7 @@ static PyTypeObject* static_types[] = { &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, + &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, &_Py_GenericAliasIterType, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 24b260d2a3cb83..6874e174460bb3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8419,6 +8419,64 @@ slot_tp_finalize(PyObject *self) PyErr_Restore(error_type, error_value, error_traceback); } +typedef struct _PyBufferWrapper { + PyObject_HEAD + PyObject *mv; + PyObject *obj; +} PyBufferWrapper; + +static int +bufferwrapper_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + Py_VISIT(bw->mv); + Py_VISIT(bw->obj); + return 0; +} + +static void +bufferwrapper_dealloc(PyObject *self) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(bw->mv); + Py_XDECREF(bw->obj); + Py_TYPE(self)->tp_free(self); +} + +static void +bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + assert(PyMemoryView_Check(bw->mv)); + Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); + if (Py_TYPE(bw->obj)->tp_as_buffer != NULL + && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { + Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + } +} + +static PyBufferProcs bufferwrapper_as_buffer = { + .bf_releasebuffer = bufferwrapper_releasebuf, +}; + + +PyTypeObject _PyBufferWrapper_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "_buffer_wrapper", + .tp_basicsize = sizeof(PyBufferWrapper), + .tp_alloc = PyType_GenericAlloc, + .tp_new = PyType_GenericNew, + .tp_free = PyObject_GC_Del, + .tp_traverse = bufferwrapper_traverse, + .tp_dealloc = bufferwrapper_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_as_buffer = &bufferwrapper_as_buffer, +}; + static int slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) { @@ -8426,6 +8484,7 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) if (flags_obj == NULL) { return -1; } + PyObject *wrapper = NULL; PyObject *stack[2] = {self, flags_obj}; PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); if (ret == NULL) { @@ -8436,13 +8495,27 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) "__buffer__ returned non-memoryview object"); goto fail; } - *buffer = ((PyMemoryViewObject *)ret)->view; - Py_INCREF(buffer->obj); + + if (PyObject_GetBuffer(ret, buffer, flags) < 0) { + goto fail; + } + assert(buffer->obj == ret); + + wrapper = PyObject_GC_New(PyBufferWrapper, &_PyBufferWrapper_Type); + if (wrapper == NULL) { + goto fail; + } + ((PyBufferWrapper *)wrapper)->mv = ret; + ((PyBufferWrapper *)wrapper)->obj = Py_NewRef(self); + _PyObject_GC_TRACK(wrapper); + + buffer->obj = wrapper; Py_DECREF(ret); Py_DECREF(flags_obj); return 0; fail: + Py_XDECREF(wrapper); Py_XDECREF(ret); Py_DECREF(flags_obj); return -1; @@ -8456,9 +8529,6 @@ slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) PyErr_WriteUnraisable(self); return; } - if (buffer->obj != NULL) { - ((PyMemoryViewObject *)mv)->view.obj = Py_NewRef(buffer->obj); - } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); Py_DECREF(mv); From ac10887613a1f067487eeb35581ec525f996c190 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Oct 2022 17:01:52 -0700 Subject: [PATCH 05/35] one more test --- Lib/test/test_collections.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index a8ea91d973638f..23483c5086d578 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1948,6 +1948,7 @@ def test_Buffer(self): for sample in [str, list, tuple]: self.assertNotIsInstance(sample(), Buffer) self.assertFalse(issubclass(sample, Buffer)) + self.validate_abstract_methods(Buffer, '__buffer__') def test_MutableBuffer(self): for sample in [bytearray, memoryview]: @@ -1956,6 +1957,7 @@ def test_MutableBuffer(self): for sample in [bytes, str, list, tuple]: self.assertNotIsInstance(sample(), MutableBuffer) self.assertFalse(issubclass(sample, MutableBuffer)) + self.validate_abstract_methods(MutableBuffer, '__buffer__', '__release_buffer__') def test_MutableSequence(self): for sample in [tuple, str, bytes]: From 530a16099087030a6f3953a86d352cb51f578138 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 5 Oct 2022 14:17:22 -0700 Subject: [PATCH 06/35] additions --- Include/pybuffer.h | 2 +- Lib/inspect.py | 22 ++++++++++++++++++++++ Lib/test/test_buffer.py | 23 +++++++++++++++++++++++ Objects/typeobject.c | 16 ++++++++++++---- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/Include/pybuffer.h b/Include/pybuffer.h index 6893505e66e3e8..7173639520fe50 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -101,7 +101,7 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 -/* Flags for getting buffers */ +/* Flags for getting buffers. Keep these in sync with inspect.BufferFlags. */ #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 diff --git a/Lib/inspect.py b/Lib/inspect.py index 498ee7ab9eaf8a..f71ae8eb53bd85 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -3273,6 +3273,28 @@ def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=F globals=globals, locals=locals, eval_str=eval_str) +class BufferFlags(enum.IntFlag): + SIMPLE = 0x0 + WRITABLE = 0x1 + FORMAT = 0x4 + ND = 0x8 + STRIDES = 0x10 | ND + C_CONTIGUOUS = 0x20 | STRIDES + F_CONTIGUOUS = 0x40 | STRIDES + ANY_CONTIGUOUS = 0x80 | STRIDES + INDIRECT = 0x100 | STRIDES + CONTIG = ND | WRITABLE + CONTIG_RO = ND + STRIDED = STRIDES | WRITABLE + STRIDED_RO = STRIDES + RECORDS = STRIDES | WRITABLE | FORMAT + RECORDS_RO = STRIDES | FORMAT + FULL = INDIRECT | WRITABLE | FORMAT + FULL_RO = INDIRECT | FORMAT + READ = 0x100 + WRITE = 0x200 + + def _main(): """ Logic for inspecting an object given at command line """ import argparse diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index c46045ae96485e..9bee30bacd3b73 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4479,6 +4479,29 @@ def __release_buffer__(self, buffer): self.assertEqual(mv.tobytes(), b"hello") self.assertFalse(wr.held) + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + def test_call_builtins(self): ba = bytearray(b"hello") mv = ba.__buffer__(0) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0bdf5ec69f390c..820fba90801818 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8526,10 +8526,18 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) { - PyObject *mv = PyMemoryView_FromBuffer(buffer); - if (mv == NULL) { - PyErr_WriteUnraisable(self); - return; + PyObject *mv; + if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + // Make sure we pass the same memoryview to + // __release_buffer__() that __buffer__() returned. + mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + } + else { + mv = PyMemoryView_FromBuffer(buffer); + if (mv == NULL) { + PyErr_WriteUnraisable(self); + return; + } } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); From 207d2fd67ebe2bdcf22348cb50e4bfdf0a33a2eb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Oct 2022 21:46:56 -0700 Subject: [PATCH 07/35] introduce __mutable_buffer__ (rather hackily) --- Include/internal/pycore_abstract.h | 4 ++++ Lib/_collections_abc.py | 7 ++----- Modules/_ctypes/_ctypes.c | 17 +++++++++++++---- Modules/_io/bytesio.c | 8 +++++++- Modules/arraymodule.c | 2 ++ Modules/mmapmodule.c | 2 ++ Objects/abstract.c | 5 +++++ Objects/bytearrayobject.c | 9 +++++++-- Objects/memoryobject.c | 3 ++- Objects/picklebufobject.c | 6 ++++++ Objects/typeobject.c | 25 +++++++++++++++++-------- 11 files changed, 67 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index b1afb2dc7be65e..6db26bf246ee7c 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -19,6 +19,10 @@ _PyIndex_Check(PyObject *obj) PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +PyAPI_FUNC(PyObject *) _PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored); + +#define _PY_BUFFER_MUTABLE_BUFFER_GETSET { "__mutable_buffer__", _PyBuffer_MutableBufferGetter, NULL, NULL }, + #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 3b0d0eeb1fa9d1..c4b3e687315658 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -457,15 +457,12 @@ def __subclasshook__(cls, C): class MutableBuffer(Buffer): __slots__ = () - - @abstractmethod - def __release_buffer__(self, buffer: memoryview, /) -> None: - pass + __mutable_buffer__ = True @classmethod def __subclasshook__(cls, C): if cls is MutableBuffer: - return _check_methods(C, "__buffer__", "__release_buffer__") + return _check_methods(C, "__buffer__", "__mutable_buffer__") return NotImplemented diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index a3c7c8c471e584..1dbd85423bb87f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -127,6 +127,7 @@ bytes(cdata) #include "ctypes.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET PyObject *PyExc_ArgError = NULL; @@ -2809,6 +2810,11 @@ static PyBufferProcs PyCData_as_buffer = { NULL, }; +static PyGetSetDef PyCData_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; + /* * CData objects are mutable, so they cannot be hashable! */ @@ -2918,7 +2924,7 @@ PyTypeObject PyCData_Type = { 0, /* tp_iternext */ PyCData_methods, /* tp_methods */ PyCData_members, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -3342,6 +3348,7 @@ static PyGetSetDef PyCFuncPtr_getsets[] = { { "argtypes", (getter)PyCFuncPtr_get_argtypes, (setter)PyCFuncPtr_set_argtypes, "specify the argument types", NULL }, + _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -4468,7 +4475,7 @@ static PyTypeObject Struct_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4510,7 +4517,7 @@ static PyTypeObject Union_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4830,7 +4837,7 @@ PyTypeObject PyCArray_Type = { 0, /* tp_iternext */ Array_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4961,6 +4968,7 @@ Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) static PyGetSetDef Simple_getsets[] = { { "value", (getter)Simple_get_value, (setter)Simple_set_value, "current value", NULL }, + _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -5204,6 +5212,7 @@ static PyGetSetDef Pointer_getsets[] = { { "contents", (getter)Pointer_get_contents, (setter)Pointer_set_contents, "the object this pointer points to (read-write)", NULL }, + _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 930ef7e29dbcf6..20e3e6a0ef7435 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_object.h" #include // offsetof() #include "_iomodule.h" @@ -1124,6 +1125,11 @@ static PyBufferProcs bytesiobuf_as_buffer = { (releasebufferproc) bytesiobuf_releasebuffer, }; +static PyGetSetDef bytesiobuf_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; + Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_io._BytesIOBuffer", /*tp_name*/ @@ -1154,7 +1160,7 @@ Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ 0, /*tp_members*/ - 0, /*tp_getset*/ + bytesiobuf_getset, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 924fbf29bfb889..98c28124909d50 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -9,6 +9,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "structmember.h" // PyMemberDef @@ -2277,6 +2278,7 @@ static PyGetSetDef array_getsets [] = { "the typecode character used to create the array"}, {"itemsize", (getter) array_get_itemsize, NULL, "the size, in bytes, of one array item"}, + _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fdce783fdec5e2..85cb0a524fb227 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -24,6 +24,7 @@ #define PY_SSIZE_T_CLEAN #include +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include "structmember.h" // PyMemberDef @@ -887,6 +888,7 @@ static struct PyMethodDef mmap_object_methods[] = { static PyGetSetDef mmap_object_getset[] = { {"closed", (getter) mmap_closed_get, NULL, NULL}, + _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 5d50491b2dd347..ea40cbe30ba1b4 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2967,3 +2967,8 @@ _Py_FreeCharPArray(char *const array[]) } PyMem_Free((void*)array); } + +PyObject * +_PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored) { + Py_RETURN_TRUE; +} diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b2962fd137d93e..a413deba623dab 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2,7 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_bytes_methods.h" #include "pycore_bytesobject.h" #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -2289,6 +2289,11 @@ Construct a mutable bytearray object from:\n\ static PyObject *bytearray_iter(PyObject *seq); +static PyGetSetDef bytearray_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; + PyTypeObject PyByteArray_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "bytearray", @@ -2320,7 +2325,7 @@ PyTypeObject PyByteArray_Type = { 0, /* tp_iternext */ bytearray_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + bytearray_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index db832aa5c9b22a..6e414ee5f52427 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -11,7 +11,7 @@ */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_strhex.h" // _Py_strhex_with_sep() #include // offsetof() @@ -3186,6 +3186,7 @@ static PyGetSetDef memory_getsetlist[] = { {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, + _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL, NULL, NULL, NULL}, }; diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index aaa852cfbb05b0..7ec04206fa47df 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -2,6 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include typedef struct { @@ -203,6 +204,10 @@ static PyMethodDef picklebuf_methods[] = { {NULL, NULL} }; +static PyGetSetDef picklebuf_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; PyTypeObject PyPickleBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "pickle.PickleBuffer", @@ -216,4 +221,5 @@ PyTypeObject PyPickleBuffer_Type = { .tp_weaklistoffset = offsetof(PyPickleBufferObject, weakreflist), .tp_as_buffer = &picklebuf_as_buffer, .tp_methods = picklebuf_methods, + .tp_getset = picklebuf_getset, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 820fba90801818..0d979a1375cec6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_call.h" +#include "pycore_abstract.h" // _PyBuffer_MutableBufferGetter #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -6094,16 +6095,24 @@ type_add_getset(PyTypeObject *type) PyObject *dict = type->tp_dict; for (; gsp->name != NULL; gsp++) { - PyObject *descr = PyDescr_NewGetSet(type, gsp); - if (descr == NULL) { - return -1; - } - - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (gsp->get == _PyBuffer_MutableBufferGetter) { + PyObject *name = PyUnicode_FromString(gsp->name); + if (PyDict_SetDefault(dict, name, Py_True) == NULL) { + Py_DECREF(name); + return -1; + } + Py_DECREF(name); + } else { + PyObject *descr = PyDescr_NewGetSet(type, gsp); + if (descr == NULL) { + return -1; + } + if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + Py_DECREF(descr); + return -1; + } Py_DECREF(descr); - return -1; } - Py_DECREF(descr); } return 0; } From 007fdc115383b42b78ddf28bdcd47714de9c53e7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 6 Nov 2022 06:47:23 -0800 Subject: [PATCH 08/35] Rip out __mutable_buffer__ --- Include/internal/pycore_abstract.h | 4 ---- Lib/_collections_abc.py | 14 +------------- Lib/test/test_collections.py | 11 +---------- Modules/_ctypes/_ctypes.c | 17 ++++------------- Modules/_io/bytesio.c | 8 +------- Modules/arraymodule.c | 2 -- Modules/mmapmodule.c | 2 -- Objects/abstract.c | 5 ----- Objects/bytearrayobject.c | 9 ++------- Objects/memoryobject.c | 3 +-- Objects/picklebufobject.c | 6 ------ Objects/typeobject.c | 30 ++++++++++-------------------- 12 files changed, 20 insertions(+), 91 deletions(-) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 6db26bf246ee7c..b1afb2dc7be65e 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -19,10 +19,6 @@ _PyIndex_Check(PyObject *obj) PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); -PyAPI_FUNC(PyObject *) _PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored); - -#define _PY_BUFFER_MUTABLE_BUFFER_GETSET { "__mutable_buffer__", _PyBuffer_MutableBufferGetter, NULL, NULL }, - #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c4b3e687315658..10ddcaf3e00134 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", "MutableBuffer", + "ByteString", "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -454,18 +454,6 @@ def __subclasshook__(cls, C): return NotImplemented -class MutableBuffer(Buffer): - - __slots__ = () - __mutable_buffer__ = True - - @classmethod - def __subclasshook__(cls, C): - if cls is MutableBuffer: - return _check_methods(C, "__buffer__", "__mutable_buffer__") - return NotImplemented - - class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index e1b6f2bafe0930..b257b66fcf6752 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -25,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer, MutableBuffer +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -1953,15 +1953,6 @@ def test_Buffer(self): self.assertFalse(issubclass(sample, Buffer)) self.validate_abstract_methods(Buffer, '__buffer__') - def test_MutableBuffer(self): - for sample in [bytearray, memoryview]: - self.assertIsInstance(sample(b"x"), MutableBuffer) - self.assertTrue(issubclass(sample, MutableBuffer)) - for sample in [bytes, str, list, tuple]: - self.assertNotIsInstance(sample(), MutableBuffer) - self.assertFalse(issubclass(sample, MutableBuffer)) - self.validate_abstract_methods(MutableBuffer, '__buffer__', '__release_buffer__') - def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 0e89ca0e079274..f342b17abca698 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -125,7 +125,6 @@ bytes(cdata) #include "ctypes.h" #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET PyObject *PyExc_ArgError = NULL; @@ -2793,11 +2792,6 @@ static PyBufferProcs PyCData_as_buffer = { NULL, }; -static PyGetSetDef PyCData_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; - /* * CData objects are mutable, so they cannot be hashable! */ @@ -2907,7 +2901,7 @@ PyTypeObject PyCData_Type = { 0, /* tp_iternext */ PyCData_methods, /* tp_methods */ PyCData_members, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -3330,7 +3324,6 @@ static PyGetSetDef PyCFuncPtr_getsets[] = { { "argtypes", (getter)PyCFuncPtr_get_argtypes, (setter)PyCFuncPtr_set_argtypes, "specify the argument types", NULL }, - _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -4455,7 +4448,7 @@ static PyTypeObject Struct_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4497,7 +4490,7 @@ static PyTypeObject Union_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4817,7 +4810,7 @@ PyTypeObject PyCArray_Type = { 0, /* tp_iternext */ Array_methods, /* tp_methods */ 0, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4948,7 +4941,6 @@ Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) static PyGetSetDef Simple_getsets[] = { { "value", (getter)Simple_get_value, (setter)Simple_set_value, "current value", NULL }, - _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -5192,7 +5184,6 @@ static PyGetSetDef Pointer_getsets[] = { { "contents", (getter)Pointer_get_contents, (setter)Pointer_set_contents, "the object this pointer points to (read-write)", NULL }, - _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 20e3e6a0ef7435..930ef7e29dbcf6 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_object.h" #include // offsetof() #include "_iomodule.h" @@ -1125,11 +1124,6 @@ static PyBufferProcs bytesiobuf_as_buffer = { (releasebufferproc) bytesiobuf_releasebuffer, }; -static PyGetSetDef bytesiobuf_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; - Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_io._BytesIOBuffer", /*tp_name*/ @@ -1160,7 +1154,7 @@ Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ 0, /*tp_members*/ - bytesiobuf_getset, /*tp_getset*/ + 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index ab690efd52dd9f..d60cf26788f5a6 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -9,7 +9,6 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "structmember.h" // PyMemberDef @@ -2278,7 +2277,6 @@ static PyGetSetDef array_getsets [] = { "the typecode character used to create the array"}, {"itemsize", (getter) array_get_itemsize, NULL, "the size, in bytes, of one array item"}, - _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 85cb0a524fb227..fdce783fdec5e2 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -24,7 +24,6 @@ #define PY_SSIZE_T_CLEAN #include -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include "structmember.h" // PyMemberDef @@ -888,7 +887,6 @@ static struct PyMethodDef mmap_object_methods[] = { static PyGetSetDef mmap_object_getset[] = { {"closed", (getter) mmap_closed_get, NULL, NULL}, - _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index ea40cbe30ba1b4..5d50491b2dd347 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2967,8 +2967,3 @@ _Py_FreeCharPArray(char *const array[]) } PyMem_Free((void*)array); } - -PyObject * -_PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored) { - Py_RETURN_TRUE; -} diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a413deba623dab..b2962fd137d93e 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2,7 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" #include "pycore_bytesobject.h" #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -2289,11 +2289,6 @@ Construct a mutable bytearray object from:\n\ static PyObject *bytearray_iter(PyObject *seq); -static PyGetSetDef bytearray_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; - PyTypeObject PyByteArray_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "bytearray", @@ -2325,7 +2320,7 @@ PyTypeObject PyByteArray_Type = { 0, /* tp_iternext */ bytearray_methods, /* tp_methods */ 0, /* tp_members */ - bytearray_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 6e414ee5f52427..db832aa5c9b22a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -11,7 +11,7 @@ */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_strhex.h" // _Py_strhex_with_sep() #include // offsetof() @@ -3186,7 +3186,6 @@ static PyGetSetDef memory_getsetlist[] = { {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, - _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL, NULL, NULL, NULL}, }; diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index 7ec04206fa47df..aaa852cfbb05b0 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -2,7 +2,6 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include typedef struct { @@ -204,10 +203,6 @@ static PyMethodDef picklebuf_methods[] = { {NULL, NULL} }; -static PyGetSetDef picklebuf_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; PyTypeObject PyPickleBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "pickle.PickleBuffer", @@ -221,5 +216,4 @@ PyTypeObject PyPickleBuffer_Type = { .tp_weaklistoffset = offsetof(PyPickleBufferObject, weakreflist), .tp_as_buffer = &picklebuf_as_buffer, .tp_methods = picklebuf_methods, - .tp_getset = picklebuf_getset, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9e3b6d81dddaac..18ffd3f26af77b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2,7 +2,6 @@ #include "Python.h" #include "pycore_call.h" -#include "pycore_abstract.h" // _PyBuffer_MutableBufferGetter #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -6196,24 +6195,15 @@ type_add_getset(PyTypeObject *type) PyObject *dict = type->tp_dict; for (; gsp->name != NULL; gsp++) { - if (gsp->get == _PyBuffer_MutableBufferGetter) { - PyObject *name = PyUnicode_FromString(gsp->name); - if (PyDict_SetDefault(dict, name, Py_True) == NULL) { - Py_DECREF(name); - return -1; - } - Py_DECREF(name); - } else { - PyObject *descr = PyDescr_NewGetSet(type, gsp); - if (descr == NULL) { - return -1; - } - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { - Py_DECREF(descr); - return -1; - } + PyObject *descr = PyDescr_NewGetSet(type, gsp); + if (descr == NULL) { + return -1; + } + if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { Py_DECREF(descr); + return -1; } + Py_DECREF(descr); } return 0; } @@ -8565,7 +8555,7 @@ bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) assert(PyMemoryView_Check(bw->mv)); Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); - if (Py_TYPE(bw->obj)->tp_as_buffer != NULL + if (Py_TYPE(bw->obj)->tp_as_buffer != NULL && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); } @@ -8589,7 +8579,7 @@ PyTypeObject _PyBufferWrapper_Type = { .tp_as_buffer = &bufferwrapper_as_buffer, }; -static int +static int slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) { PyObject *flags_obj = PyLong_FromLong(flags); @@ -8837,7 +8827,7 @@ static slotdef slotdefs[] = { "Return a buffer object that exposes the underlying memory of the object."), BUFSLOT(__release_buffer__, bf_releasebuffer, slot_bf_releasebuffer, wrap_releasebuffer, "__release_buffer__($self, /)\n--\n\n" - "Release the buffer object that exposes the underlying memory of the object."), + "Release the buffer object that exposes the underlying memory of the object."), AMSLOT(__await__, am_await, slot_am_await, wrap_unaryfunc, "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."), From 46a9239265c62ac07bdc8127ad74995d0720d161 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 7 Nov 2022 20:14:38 -0800 Subject: [PATCH 09/35] __release_buffer__ calls mv.release() --- Objects/typeobject.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 18ffd3f26af77b..8daa92bf327cfc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7685,7 +7685,16 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) "memoryview's buffer is not this object"); return NULL; } - (*func)(self, &mview->view); + PyObject *release = PyUnicode_FromString("release"); + if (release == NULL) { + return NULL; + } + PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, release); + Py_DECREF(release); + if (res == NULL) { + return NULL; + } + Py_DECREF(res); Py_RETURN_NONE; } From be9bf456fb82dd9d15870a33ae0de23c04b4fb52 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 7 Nov 2022 20:51:53 -0800 Subject: [PATCH 10/35] additional test --- Lib/test/test_buffer.py | 16 ++++++++ Modules/_testcapimodule.c | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 9bee30bacd3b73..3e48cf48b791d3 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4508,6 +4508,22 @@ def test_call_builtins(self): self.assertEqual(mv.tobytes(), b"hello") ba.__release_buffer__(mv) + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0e09c97bea09a5..d61b6649118337 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6804,6 +6804,85 @@ static PyTypeObject MyList_Type = { MyList_new, /* tp_new */ }; +typedef struct { + PyObject_HEAD + PyObject *obj; + long references; +} testBufObject; + +static PyObject * +testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + self->obj = PyBytes_FromString("test"); + self->references = 0; + return (PyObject *)self; +} + +static int +testbuf_traverse(testBufObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->obj); + return 0; +} + +static int +testbuf_clear(testBufObject *self) +{ + Py_CLEAR(self->obj); + return 0; +} + +static void +testbuf_dealloc(testBufObject *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->obj); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static int +testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) +{ + int buf = PyObject_GetBuffer(self->obj, view, flags); + Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); + self->references++; + return buf; +} + +static void +testbuf_releasebuf(testBufObject *self, Py_buffer *view) +{ + self->references--; + assert(self->references >= 0); +} + +static PyBufferProcs testbuf_as_buffer = { + .bf_getbuffer = (getbufferproc) testbuf_getbuf, + .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, +}; + +static struct PyMemberDef testbuf_members[] = { + {"references", T_LONG, offsetof(testBufObject, references), READONLY}, + {NULL}, +}; + +static PyTypeObject testBufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "testBufType", + .tp_basicsize = sizeof(testBufObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_new = testbuf_new, + .tp_dealloc = (destructor) testbuf_dealloc, + .tp_traverse = (traverseproc) testbuf_traverse, + .tp_clear = (inquiry) testbuf_clear, + .tp_as_buffer = &testbuf_as_buffer, + .tp_members = testbuf_members +}; + /* Test PEP 560 */ @@ -7042,6 +7121,13 @@ PyInit__testcapi(void) Py_INCREF(&MyList_Type); PyModule_AddObject(m, "MyList", (PyObject *)&MyList_Type); + if (PyType_Ready(&testBufType) < 0) { + return NULL; + } + if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { + return NULL; + } + if (PyType_Ready(&GenericAlias_Type) < 0) return NULL; Py_INCREF(&GenericAlias_Type); From a6bf0e81c7cd6b6e086ba98c07f3baefc3cb6f9f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 7 Nov 2022 20:58:28 -0800 Subject: [PATCH 11/35] undo stray change --- Objects/typeobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8daa92bf327cfc..b0cf8fb814a245 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6199,6 +6199,7 @@ type_add_getset(PyTypeObject *type) if (descr == NULL) { return -1; } + if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { Py_DECREF(descr); return -1; From 04d0a420fabebc893cecf12abcb8245980ab1e4d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 17 Dec 2022 20:37:36 -0800 Subject: [PATCH 12/35] throw an error if already released --- Include/internal/pycore_global_objects_fini_generated.h | 2 ++ Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Lib/test/test_buffer.py | 3 ++- Objects/typeobject.c | 6 +++++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6aba2f19ebde4a..e9968a07ade0e8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -594,6 +594,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__build_class__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__builtins__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bytes__)); @@ -693,6 +694,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rdivmod__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce_ex__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__release_buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__repr__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reversed__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rfloordiv__)); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7f407c0141b8a5..bd1e7196b964ac 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -66,6 +66,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__bool__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__build_class__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__builtins__); @@ -264,6 +266,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce_ex__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__release_buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__repr__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reversed__); diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 3e48cf48b791d3..8b2d7e12fce17b 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4521,7 +4521,8 @@ def test_c_buffer(self): with self.assertRaises(ValueError): mv.tobytes() # Calling it again doesn't cause issues - buf.__release_buffer__(mv) + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 95a740c94c9858..7f24503f6f714c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7624,7 +7624,6 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) static PyObject * wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) { - releasebufferproc func = (releasebufferproc)wrapped; PyMemoryViewObject *mview; if (!check_num_args(args, 1)) { @@ -7638,6 +7637,11 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) "memoryview's buffer is not this object"); return NULL; } + if (mview->flags & _Py_MEMORYVIEW_RELEASED) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer has already been released"); + return NULL; + } PyObject *release = PyUnicode_FromString("release"); if (release == NULL) { return NULL; From 5f755fec0da074bf881c6deb7ea4af71e86cef0f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Mar 2023 17:35:00 -0800 Subject: [PATCH 13/35] Fix compiler warning --- Objects/typeobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f09b3acbc39d35..96282cb2bad905 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8579,7 +8579,7 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) if (flags_obj == NULL) { return -1; } - PyObject *wrapper = NULL; + PyBufferWrapper *wrapper = NULL; PyObject *stack[2] = {self, flags_obj}; PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); if (ret == NULL) { @@ -8600,11 +8600,11 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) if (wrapper == NULL) { goto fail; } - ((PyBufferWrapper *)wrapper)->mv = ret; - ((PyBufferWrapper *)wrapper)->obj = Py_NewRef(self); + wrapper->mv = ret; + wrapper->obj = Py_NewRef(self); _PyObject_GC_TRACK(wrapper); - buffer->obj = wrapper; + buffer->obj = (PyObject *)wrapper; Py_DECREF(ret); Py_DECREF(flags_obj); return 0; From 0b02e63d23961ab1f67093415cb35ef34f8a82e8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Mar 2023 17:37:06 -0800 Subject: [PATCH 14/35] news --- .../2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst new file mode 100644 index 00000000000000..e03113ba05cd7d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst @@ -0,0 +1,3 @@ +Make the buffer protocol accessible in Python code using the new +``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for +details. Patch by Jelle Zijlstra. From 8e4db4300e3137fd608bc24bfcde3c0098a426e7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Mar 2023 18:19:13 -0800 Subject: [PATCH 15/35] fix some tests --- Lib/inspect.py | 1 + Lib/test/test_doctest.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index 9ab7d61df3a630..86f6f499242c76 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -39,6 +39,7 @@ "Attribute", "BlockFinder", "BoundArguments", + "BufferFlags", "CORO_CLOSED", "CORO_CREATED", "CORO_RUNNING", diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3491d4cdb1c18b..542fcdb5cf6f66 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 850 # approximate number of objects with docstrings + >>> 830 < len(tests) < 860 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests From 6c863bcf3560563a736ebcefba12c9155a6f7869 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 Mar 2023 20:58:41 -0800 Subject: [PATCH 16/35] More tests. Add flags= argument to memoryview --- Lib/test/test_buffer.py | 35 +++++++++++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/buffer.c | 97 +++++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 90 +--------------------------- Objects/clinic/memoryobject.c.h | 26 ++++++--- Objects/memoryobject.c | 8 ++- PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + 9 files changed, 164 insertions(+), 99 deletions(-) create mode 100644 Modules/_testcapi/buffer.c diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 8b2d7e12fce17b..914cb58675b2b7 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,6 +17,7 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings @@ -4502,6 +4503,40 @@ def __release_buffer__(self, buffer): self.assertEqual(mv.tobytes(), b"hello") self.assertFalse(wr.held) + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview(mutable, flags=inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview(immutable, flags=inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview(immutable, flags=inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + def test_call_builtins(self): ba = bytearray(b"hello") mv = ba.__buffer__(0) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b12290d436cbeb..c5c231ccb33625 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c new file mode 100644 index 00000000000000..e625e941847c37 --- /dev/null +++ b/Modules/_testcapi/buffer.c @@ -0,0 +1,97 @@ +/* Test PEP 688 - Buffers */ + +#include "parts.h" + +#include "structmember.h" // PyMemberDef +#include // offsetof + +typedef struct { + PyObject_HEAD + PyObject *obj; + long references; +} testBufObject; + +static PyObject * +testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + self->obj = PyBytes_FromString("test"); + self->references = 0; + return (PyObject *)self; +} + +static int +testbuf_traverse(testBufObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->obj); + return 0; +} + +static int +testbuf_clear(testBufObject *self) +{ + Py_CLEAR(self->obj); + return 0; +} + +static void +testbuf_dealloc(testBufObject *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->obj); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static int +testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) +{ + int buf = PyObject_GetBuffer(self->obj, view, flags); + Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); + self->references++; + return buf; +} + +static void +testbuf_releasebuf(testBufObject *self, Py_buffer *view) +{ + self->references--; + assert(self->references >= 0); +} + +static PyBufferProcs testbuf_as_buffer = { + .bf_getbuffer = (getbufferproc) testbuf_getbuf, + .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, +}; + +static struct PyMemberDef testbuf_members[] = { + {"references", T_LONG, offsetof(testBufObject, references), READONLY}, + {NULL}, +}; + +static PyTypeObject testBufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "testBufType", + .tp_basicsize = sizeof(testBufObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_new = testbuf_new, + .tp_dealloc = (destructor) testbuf_dealloc, + .tp_traverse = (traverseproc) testbuf_traverse, + .tp_clear = (inquiry) testbuf_clear, + .tp_as_buffer = &testbuf_as_buffer, + .tp_members = testbuf_members +}; + +int +_PyTestCapi_Init_Buffer(PyObject *m) { + if (PyType_Ready(&testBufType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index c8f31dc8e39fae..9348ab382344f1 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -38,6 +38,7 @@ int _PyTestCapi_Init_Float(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); +int _PyTestCapi_Init_Buffer(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f9ede10dec8eed..0ba964d71bcde3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3746,86 +3746,6 @@ static PyTypeObject MyList_Type = { MyList_new, /* tp_new */ }; -typedef struct { - PyObject_HEAD - PyObject *obj; - long references; -} testBufObject; - -static PyObject * -testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - self->obj = PyBytes_FromString("test"); - self->references = 0; - return (PyObject *)self; -} - -static int -testbuf_traverse(testBufObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->obj); - return 0; -} - -static int -testbuf_clear(testBufObject *self) -{ - Py_CLEAR(self->obj); - return 0; -} - -static void -testbuf_dealloc(testBufObject *self) -{ - PyObject_GC_UnTrack(self); - Py_XDECREF(self->obj); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static int -testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) -{ - int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); - self->references++; - return buf; -} - -static void -testbuf_releasebuf(testBufObject *self, Py_buffer *view) -{ - self->references--; - assert(self->references >= 0); -} - -static PyBufferProcs testbuf_as_buffer = { - .bf_getbuffer = (getbufferproc) testbuf_getbuf, - .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, -}; - -static struct PyMemberDef testbuf_members[] = { - {"references", T_LONG, offsetof(testBufObject, references), READONLY}, - {NULL}, -}; - -static PyTypeObject testBufType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "testBufType", - .tp_basicsize = sizeof(testBufObject), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_new = testbuf_new, - .tp_dealloc = (destructor) testbuf_dealloc, - .tp_traverse = (traverseproc) testbuf_traverse, - .tp_clear = (inquiry) testbuf_clear, - .tp_as_buffer = &testbuf_as_buffer, - .tp_members = testbuf_members -}; - - /* Test PEP 560 */ typedef struct { @@ -4057,13 +3977,6 @@ PyInit__testcapi(void) Py_INCREF(&MyList_Type); PyModule_AddObject(m, "MyList", (PyObject *)&MyList_Type); - if (PyType_Ready(&testBufType) < 0) { - return NULL; - } - if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { - return NULL; - } - if (PyType_Ready(&GenericAlias_Type) < 0) return NULL; Py_INCREF(&GenericAlias_Type); @@ -4172,6 +4085,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Code(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Buffer(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index ff7b50bb114b05..1e7ee55ab960ae 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -9,13 +9,13 @@ preserve PyDoc_STRVAR(memoryview__doc__, -"memoryview(object)\n" +"memoryview(object, *, flags=PyBUF_FULL_RO)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object); +memoryview_impl(PyTypeObject *type, PyObject *object, int flags); static PyObject * memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -23,14 +23,14 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), }, + .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -39,24 +39,34 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", NULL}; + static const char * const _keywords[] = {"object", "flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "memoryview", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *object; + int flags = PyBUF_FULL_RO; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { goto exit; } object = fastargs[0]; - return_value = memoryview_impl(type, object); + if (!noptargs) { + goto skip_optional_kwonly; + } + flags = _PyLong_AsInt(fastargs[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = memoryview_impl(type, object, flags); exit: return return_value; @@ -356,4 +366,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=a832f2fc44e4794c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9de9bd412f8fea52 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 9d69a615c537bc..6e8384ce4401bd 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -965,15 +965,17 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) memoryview.__new__ object: object + * + flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object) -/*[clinic end generated code: output=7de78e184ed66db8 input=f04429eb0bdf8c6e]*/ +memoryview_impl(PyTypeObject *type, PyObject *object, int flags) +/*[clinic end generated code: output=f1c225d1ad24ec11 input=cab66ab4bb27f268]*/ { - return PyMemoryView_FromObject(object); + return PyMemoryView_FromObjectAndFlags(object, flags); } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 4cc184bfc1ac82..6be0aacf57074a 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -109,6 +109,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index fbdaf04ce37cb1..cc4b22361b328a 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -57,6 +57,9 @@ Source Files + + Source Files + From 3b4b7d69587ef4a551798560b904246446cd9404 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 10 Mar 2023 16:54:51 -0800 Subject: [PATCH 17/35] Make memoryview flags arg private --- Lib/test/test_buffer.py | 6 +++--- Objects/clinic/memoryobject.c.h | 18 +++++++++--------- Objects/memoryobject.c | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 914cb58675b2b7..655ecf42176df8 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4519,18 +4519,18 @@ def __buffer__(self, flags): mutable = PossiblyMutable(b"hello", True) immutable = PossiblyMutable(b"hello", False) - with memoryview(mutable, flags=inspect.BufferFlags.WRITABLE) as mv: + with memoryview(mutable, _flags=inspect.BufferFlags.WRITABLE) as mv: self.assertEqual(mv.tobytes(), b"hello") mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"xello") - with memoryview(immutable, flags=inspect.BufferFlags.SIMPLE) as mv: + with memoryview(immutable, _flags=inspect.BufferFlags.SIMPLE) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(RuntimeError): - memoryview(immutable, flags=inspect.BufferFlags.WRITABLE) + memoryview(immutable, _flags=inspect.BufferFlags.WRITABLE) with memoryview(immutable) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 1e7ee55ab960ae..26f96fc91b832e 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -9,13 +9,13 @@ preserve PyDoc_STRVAR(memoryview__doc__, -"memoryview(object, *, flags=PyBUF_FULL_RO)\n" +"memoryview(object, *, _flags=PyBUF_FULL_RO)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int flags); +memoryview_impl(PyTypeObject *type, PyObject *object, int _flags); static PyObject * memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -30,7 +30,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, + .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -39,7 +39,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", "flags", NULL}; + static const char * const _keywords[] = {"object", "_flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "memoryview", @@ -51,7 +51,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *object; - int flags = PyBUF_FULL_RO; + int _flags = PyBUF_FULL_RO; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -61,12 +61,12 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_kwonly; } - flags = _PyLong_AsInt(fastargs[1]); - if (flags == -1 && PyErr_Occurred()) { + _flags = _PyLong_AsInt(fastargs[1]); + if (_flags == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = memoryview_impl(type, object, flags); + return_value = memoryview_impl(type, object, _flags); exit: return return_value; @@ -366,4 +366,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=9de9bd412f8fea52 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=392fda9a93c25dd2 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 6e8384ce4401bd..f119bb5d36d6d1 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -966,16 +966,16 @@ memoryview.__new__ object: object * - flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO + _flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int flags) -/*[clinic end generated code: output=f1c225d1ad24ec11 input=cab66ab4bb27f268]*/ +memoryview_impl(PyTypeObject *type, PyObject *object, int _flags) +/*[clinic end generated code: output=80388b83c45dafac input=2ea9a227b7f350e8]*/ { - return PyMemoryView_FromObjectAndFlags(object, flags); + return PyMemoryView_FromObjectAndFlags(object, _flags); } From f2950127b84c1888f7db07b56c71d2be0f377064 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 10 Mar 2023 17:05:59 -0800 Subject: [PATCH 18/35] regen global objects --- Include/internal/pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init_generated.h | 1 + Include/internal/pycore_unicodeobject_generated.h | 2 ++ 4 files changed, 5 insertions(+) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 87d2700f991f2c..f60d005dad71a6 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -751,6 +751,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fb9fefef0db37a..2f431527d69d9f 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -237,6 +237,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) STRUCT_FOR_ID(_fix_up_module) + STRUCT_FOR_ID(_flags) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 77f8f464ad604c..f6d1e2c9095ebd 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -743,6 +743,7 @@ extern "C" { INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ INIT_ID(_fix_up_module), \ + INIT_ID(_flags), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f59f0e716d859c..e6e861f5d4e9e0 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -380,6 +380,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_fix_up_module); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_flags); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_flags_); PyUnicode_InternInPlace(&string); string = &_Py_ID(_get_sourcefile); From b5ea9088b8c146faddb28f4bd5d4417e1e0124c5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Apr 2023 18:13:13 -0700 Subject: [PATCH 19/35] Ignore new C globals for now --- Objects/typeobject.c | 2 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 358d1a93eeebbf..b99a008ca65540 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8587,7 +8587,7 @@ static PyBufferProcs bufferwrapper_as_buffer = { }; -PyTypeObject _PyBufferWrapper_Type = { +static PyTypeObject _PyBufferWrapper_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "_buffer_wrapper", .tp_basicsize = sizeof(PyBufferWrapper), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 0620c7e13925b5..50a32d04b2d285 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -86,6 +86,7 @@ Objects/sliceobject.c - PyEllipsis_Type - Objects/sliceobject.c - PySlice_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - +Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a8ba88efc732fb..58dec8f6bb8556 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -403,6 +403,7 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - From b22bfa95ed7a9220d8767853db3563e9505d09ac Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Apr 2023 18:14:15 -0700 Subject: [PATCH 20/35] not static (should not have committed this) --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b99a008ca65540..358d1a93eeebbf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8587,7 +8587,7 @@ static PyBufferProcs bufferwrapper_as_buffer = { }; -static PyTypeObject _PyBufferWrapper_Type = { +PyTypeObject _PyBufferWrapper_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "_buffer_wrapper", .tp_basicsize = sizeof(PyBufferWrapper), From 69e8f7c9b6bddb0188e3dba4834b10b82aac8acd Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 5 Apr 2023 06:57:20 -0700 Subject: [PATCH 21/35] Use tabs not spaces --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 50a32d04b2d285..79ca4dc3e2969d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -86,7 +86,7 @@ Objects/sliceobject.c - PyEllipsis_Type - Objects/sliceobject.c - PySlice_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - -Objects/typeobject.c - _PyBufferWrapper_Type - +Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 58dec8f6bb8556..f3f8f831c48044 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -403,7 +403,7 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - -Modules/_testcapi/buffer.c - testBufType - +Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - From 4ca7a7c5f97ffd20eee255abeddb7650dd67cee4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 9 Apr 2023 21:17:32 -0700 Subject: [PATCH 22/35] Address Kumar's feedback --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 +++ Modules/_testcapi/buffer.c | 9 +++++-- Objects/typeobject.c | 25 +++++++++---------- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 8294f5764c83b4..aa8055bc3ec370 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1123,6 +1123,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(release)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 278c69ca687327..3babba2196753b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -609,6 +609,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) + STRUCT_FOR_ID(release) STRUCT_FOR_ID(reload) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 62e0575fbcdbbe..4a8f280be337af 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1115,6 +1115,7 @@ extern "C" { INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ + INIT_ID(release), \ INIT_ID(reload), \ INIT_ID(repl), \ INIT_ID(replace), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 168e6147345c3c..f562a8c7bc147a 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1680,6 +1680,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(rel_tol); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(release); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(reload); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index e625e941847c37..29ca2c7fd91448 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -8,17 +8,22 @@ typedef struct { PyObject_HEAD PyObject *obj; - long references; + Py_ssize_t references; } testBufObject; static PyObject * testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + PyObject *obj = PyBytes_FromString("test"); + if (obj == NULL) { + return NULL; + } testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); if (self == NULL) { + Py_DECREF(obj); return NULL; } - self->obj = PyBytes_FromString("test"); + self->obj = obj; self->references = 0; return (PyObject *)self; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 358d1a93eeebbf..3d0c91e1cdd5ab 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7665,28 +7665,32 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) static PyObject * wrap_buffer(PyObject *self, PyObject *args, void *wrapped) { - int flags = 0; + PyObject *arg = NULL; - if (!check_num_args(args, 1)) { + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { return NULL; } - if (!PyArg_ParseTuple(args, "i", &flags)) { + Py_ssize_t flags = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (flags == -1 && PyErr_Occurred()) { return NULL; } + return PyMemoryView_FromObjectAndFlags(self, flags); } static PyObject * wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) { - PyMemoryViewObject *mview; - - if (!check_num_args(args, 1)) { + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { return NULL; } - if (!PyArg_ParseTuple(args, "O!", &PyMemoryView_Type, &mview)) { + if (!PyMemoryView_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "expected a memoryview object"); return NULL; } + PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; if (mview->view.obj != self) { PyErr_SetString(PyExc_ValueError, "memoryview's buffer is not this object"); @@ -7697,12 +7701,7 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) "memoryview's buffer has already been released"); return NULL; } - PyObject *release = PyUnicode_FromString("release"); - if (release == NULL) { - return NULL; - } - PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, release); - Py_DECREF(release); + PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, &_Py_ID(release)); if (res == NULL) { return NULL; } From a70e12d3206d1b76ea4069c82c9d437e3b84c468 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 12 Apr 2023 17:57:08 -0700 Subject: [PATCH 23/35] Address another piece of feedback --- Objects/typeobject.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9d0c67a1f5a969..b99a45f2aa77a1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8545,12 +8545,10 @@ typedef struct _PyBufferWrapper { } PyBufferWrapper; static int -bufferwrapper_traverse(PyObject *self, visitproc visit, void *arg) +bufferwrapper_traverse(PyBufferWrapper *self, visitproc visit, void *arg) { - PyBufferWrapper *bw = (PyBufferWrapper *)self; - - Py_VISIT(bw->mv); - Py_VISIT(bw->obj); + Py_VISIT(self->mv); + Py_VISIT(self->obj); return 0; } @@ -8590,7 +8588,7 @@ PyTypeObject _PyBufferWrapper_Type = { .tp_alloc = PyType_GenericAlloc, .tp_new = PyType_GenericNew, .tp_free = PyObject_GC_Del, - .tp_traverse = bufferwrapper_traverse, + .tp_traverse = (traverseproc)bufferwrapper_traverse, .tp_dealloc = bufferwrapper_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_as_buffer = &bufferwrapper_as_buffer, From 96c9253833528dcdfb42e277a23ecab826c5cb36 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 12 Apr 2023 18:10:50 -0700 Subject: [PATCH 24/35] Use a classmethod instead of a new arg to the memoryview constructor --- Lib/test/test_buffer.py | 6 +-- Objects/clinic/memoryobject.c.h | 78 +++++++++++++++++++++++++++------ Objects/memoryobject.c | 25 +++++++++-- 3 files changed, 88 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 655ecf42176df8..4f9e655f2612ff 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4519,18 +4519,18 @@ def __buffer__(self, flags): mutable = PossiblyMutable(b"hello", True) immutable = PossiblyMutable(b"hello", False) - with memoryview(mutable, _flags=inspect.BufferFlags.WRITABLE) as mv: + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: self.assertEqual(mv.tobytes(), b"hello") mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"xello") - with memoryview(immutable, _flags=inspect.BufferFlags.SIMPLE) as mv: + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(RuntimeError): - memoryview(immutable, _flags=inspect.BufferFlags.WRITABLE) + memoryview_from_flags(immutable, inspect.BufferFlags.WRITABLE) with memoryview(immutable) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 26f96fc91b832e..ead1af4f3e8619 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -9,13 +9,13 @@ preserve PyDoc_STRVAR(memoryview__doc__, -"memoryview(object, *, _flags=PyBUF_FULL_RO)\n" +"memoryview(object)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int _flags); +memoryview_impl(PyTypeObject *type, PyObject *object); static PyObject * memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -23,14 +23,14 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, + .ob_item = { &_Py_ID(object), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -39,34 +39,84 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", "_flags", NULL}; + static const char * const _keywords[] = {"object", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "memoryview", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[1]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *object; - int _flags = PyBUF_FULL_RO; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { goto exit; } object = fastargs[0]; - if (!noptargs) { - goto skip_optional_kwonly; + return_value = memoryview_impl(type, object); + +exit: + return return_value; +} + +PyDoc_STRVAR(memoryview__from_flags__doc__, +"_from_flags($type, /, object, _flags)\n" +"--\n" +"\n" +"Create a new memoryview object which references the given object."); + +#define MEMORYVIEW__FROM_FLAGS_METHODDEF \ + {"_from_flags", _PyCFunction_CAST(memoryview__from_flags), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, memoryview__from_flags__doc__}, + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags); + +static PyObject * +memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"object", "_flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_from_flags", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *object; + int _flags; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; } - _flags = _PyLong_AsInt(fastargs[1]); + object = args[0]; + _flags = _PyLong_AsInt(args[1]); if (_flags == -1 && PyErr_Occurred()) { goto exit; } -skip_optional_kwonly: - return_value = memoryview_impl(type, object, _flags); + return_value = memoryview__from_flags_impl(type, object, _flags); exit: return return_value; @@ -366,4 +416,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=392fda9a93c25dd2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31ed11a9a4ac95c4 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f119bb5d36d6d1..9117c3236baeb7 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -965,15 +965,31 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) memoryview.__new__ object: object - * - _flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int _flags) -/*[clinic end generated code: output=80388b83c45dafac input=2ea9a227b7f350e8]*/ +memoryview_impl(PyTypeObject *type, PyObject *object) +/*[clinic end generated code: output=7de78e184ed66db8 input=f04429eb0bdf8c6e]*/ +{ + return PyMemoryView_FromObject(object); +} + + +/*[clinic input] +@classmethod +memoryview._from_flags + + object: object + _flags: int + +Create a new memoryview object which references the given object. +[clinic start generated code]*/ + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags) +/*[clinic end generated code: output=bc1f4ba7d9b64525 input=860e7832e8edb1e5]*/ { return PyMemoryView_FromObjectAndFlags(object, _flags); } @@ -3191,6 +3207,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_TOLIST_METHODDEF MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF + MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, NULL}, {NULL, NULL} From 1f7f7a090be30a2806976385da3c050b5afe2ef1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 13 Apr 2023 12:05:28 -0700 Subject: [PATCH 25/35] fix typo --- Lib/test/test_buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 4f9e655f2612ff..e0b3e81af63071 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4530,7 +4530,7 @@ def __buffer__(self, flags): self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(RuntimeError): - memoryview_from_flags(immutable, inspect.BufferFlags.WRITABLE) + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) with memoryview(immutable) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): From 33691eab4e61bea7e55f2cdb52f1bfe71ab8a34c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 26 Apr 2023 06:22:49 -0600 Subject: [PATCH 26/35] Add Py_SAFE_DOWNCAST --- Include/internal/pycore_global_objects_fini_generated.h | 3 +++ Include/internal/pycore_global_strings.h | 3 +++ Include/internal/pycore_runtime_init_generated.h | 3 +++ Include/internal/pycore_unicodeobject_generated.h | 9 +++++++++ Objects/typeobject.c | 2 +- 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b157724b8ab30a..a79ea65b5bf0dd 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -780,8 +780,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -1076,6 +1078,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(optimize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3babba2196753b..34f89e3c25d49a 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -268,8 +268,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) + STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) + STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -564,6 +566,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(optimize) STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) + STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4a8f280be337af..c7c61ebbd29cbd 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -774,8 +774,10 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ + INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ + INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -1070,6 +1072,7 @@ extern "C" { INIT_ID(optimize), \ INIT_ID(options), \ INIT_ID(order), \ + INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(overlapped), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f562a8c7bc147a..43578138c8ba5b 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -657,12 +657,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(alias); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(args); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1545,6 +1551,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(order); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(origin); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(out_fd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 73bd3e3b31d594..0ecad2341fbca4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7698,7 +7698,7 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } - return PyMemoryView_FromObjectAndFlags(self, flags); + return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); } static PyObject * From 8f5073f400d35df3eb4b188257344b5289866f16 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 26 Apr 2023 15:42:00 -0600 Subject: [PATCH 27/35] Add some test cases (thanks Shantanu) --- Lib/test/test_buffer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index c09d923274d528..6a2b53892280bc 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4456,6 +4456,12 @@ def __buffer__(self, flags): self.assertRaises(TypeError, memoryview, MustReturnMV()) + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + class WrongArity: def __buffer__(self): return memoryview(b"hello") @@ -4525,6 +4531,11 @@ def __buffer__(self, flags): self.assertEqual(mv.tobytes(), b"hello") mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): From cecb6a58027cd6b73519cfb97da5a34ce9d19f48 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 28 Apr 2023 20:44:29 -0700 Subject: [PATCH 28/35] Remove spurious global strings --- Include/internal/pycore_global_objects_fini_generated.h | 3 --- Include/internal/pycore_global_strings.h | 3 --- Include/internal/pycore_runtime_init_generated.h | 3 --- Include/internal/pycore_unicodeobject_generated.h | 9 --------- 4 files changed, 18 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ae17f28c6b9e60..f3acefd5aca13e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -780,10 +780,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -1079,7 +1077,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(optimize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 033faa9752347e..e250528c91379b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -268,10 +268,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) - STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) - STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -567,7 +565,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(optimize) STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) - STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index ecf13a054bbd7e..fcf12e28476ae6 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -774,10 +774,8 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ - INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ - INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -1073,7 +1071,6 @@ extern "C" { INIT_ID(optimize), \ INIT_ID(options), \ INIT_ID(order), \ - INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(overlapped), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index d2fd3382c78e0c..c5720add585ea9 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -657,18 +657,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(alias); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(args); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1554,9 +1548,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(order); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(origin); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(out_fd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); From 0f77bbb8331e057f33062a6d18a16215df9faa34 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 4 May 2023 16:52:17 +0530 Subject: [PATCH 29/35] fixup global objects --- Tools/build/generate_global_objects.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index c27817702bf97d..ded19ee489e79b 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -121,6 +121,8 @@ '__xor__', '__divmod__', '__rdivmod__', + '__buffer__', + '__release_buffer__', ] NON_GENERATED_IMMORTAL_OBJECTS = [ From 61f54ced1d015c43a6d5e4223b6d1aa43356beb6 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 4 May 2023 17:02:22 +0530 Subject: [PATCH 30/35] minor fixes --- Modules/_testcapi/buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 29ca2c7fd91448..aff9a477eff57e 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -54,7 +54,7 @@ static int testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) { int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); + Py_SETREF(view->obj, Py_NewRef(self)); self->references++; return buf; } @@ -72,7 +72,7 @@ static PyBufferProcs testbuf_as_buffer = { }; static struct PyMemberDef testbuf_members[] = { - {"references", T_LONG, offsetof(testBufObject, references), READONLY}, + {"references", T_PYSSIZET, offsetof(testBufObject, references), READONLY}, {NULL}, }; From 9f2d16ba327dd3ffc5981940cb6acfac7aa50140 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 05:59:35 -0700 Subject: [PATCH 31/35] newlines --- Lib/test/test_buffer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 6a2b53892280bc..ffe40ca9ea2633 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4473,11 +4473,13 @@ class WhatToRelease: def __init__(self): self.held = False self.ba = bytearray(b"hello") + def __buffer__(self, flags): if self.held: raise TypeError("already held") self.held = True return memoryview(self.ba) + def __release_buffer__(self, buffer): self.held = False @@ -4494,12 +4496,14 @@ def __init__(self): self.held = False self.ba = bytearray(b"hello") self.created_mv = None + def __buffer__(self, flags): if self.held: raise TypeError("already held") self.held = True self.created_mv = memoryview(self.ba) return self.created_mv + def __release_buffer__(self, buffer): assert buffer is self.created_mv self.held = False From 5e0d2de8d34d10c0cb6adf2867493bc2fed8d2b2 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:09:04 -0700 Subject: [PATCH 32/35] Rename variable --- Objects/clinic/memoryobject.c.h | 18 +++++++++--------- Objects/memoryobject.c | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index ead1af4f3e8619..25a22341185903 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -63,7 +63,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(memoryview__from_flags__doc__, -"_from_flags($type, /, object, _flags)\n" +"_from_flags($type, /, object, flags)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); @@ -72,7 +72,7 @@ PyDoc_STRVAR(memoryview__from_flags__doc__, {"_from_flags", _PyCFunction_CAST(memoryview__from_flags), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, memoryview__from_flags__doc__}, static PyObject * -memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags); +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags); static PyObject * memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -87,7 +87,7 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, + .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -96,7 +96,7 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", "_flags", NULL}; + static const char * const _keywords[] = {"object", "flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "_from_flags", @@ -105,18 +105,18 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar #undef KWTUPLE PyObject *argsbuf[2]; PyObject *object; - int _flags; + int flags; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); if (!args) { goto exit; } object = args[0]; - _flags = _PyLong_AsInt(args[1]); - if (_flags == -1 && PyErr_Occurred()) { + flags = _PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { goto exit; } - return_value = memoryview__from_flags_impl(type, object, _flags); + return_value = memoryview__from_flags_impl(type, object, flags); exit: return return_value; @@ -416,4 +416,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=31ed11a9a4ac95c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=01613814112cedd7 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 8c913bcdac9cf9..f008a8cc3e0474 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -982,16 +982,16 @@ memoryview_impl(PyTypeObject *type, PyObject *object) memoryview._from_flags object: object - _flags: int + flags: int Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags) -/*[clinic end generated code: output=bc1f4ba7d9b64525 input=860e7832e8edb1e5]*/ +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags) +/*[clinic end generated code: output=bf71f9906c266ee2 input=f5f82fd0e744356b]*/ { - return PyMemoryView_FromObjectAndFlags(object, _flags); + return PyMemoryView_FromObjectAndFlags(object, flags); } From b3806322bcebca57c8f4f942c70b00a989223ee7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:11:34 -0700 Subject: [PATCH 33/35] Check for INT_MAX --- Lib/test/test_buffer.py | 2 ++ Objects/typeobject.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index ffe40ca9ea2633..b6e82ad4db266a 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4559,6 +4559,8 @@ def test_call_builtins(self): mv = ba.__buffer__(0) self.assertEqual(mv.tobytes(), b"hello") ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_c_buffer(self): diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f7ad15f70baf29..500e3d782a3eea 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8072,6 +8072,11 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) if (flags == -1 && PyErr_Occurred()) { return NULL; } + if (flags > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "buffer flags too large"); + return NULL; + } return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); } From 8502b98121d174f2c12afdb2b06f46fa17a58455 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:17:05 -0700 Subject: [PATCH 34/35] Remove tp_new --- Objects/typeobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 500e3d782a3eea..456b10ee01d6bc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8996,7 +8996,6 @@ PyTypeObject _PyBufferWrapper_Type = { .tp_name = "_buffer_wrapper", .tp_basicsize = sizeof(PyBufferWrapper), .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, .tp_free = PyObject_GC_Del, .tp_traverse = (traverseproc)bufferwrapper_traverse, .tp_dealloc = bufferwrapper_dealloc, From b51465fb3065aa581113c884c49de604aeaeeae9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:21:32 -0700 Subject: [PATCH 35/35] regen-all --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 3 --- 4 files changed, 6 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9d97da01d338e5..9377fd8526e3a2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -749,7 +749,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f7dc0010316813..ed9b2bb44ddffc 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -237,7 +237,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) STRUCT_FOR_ID(_fix_up_module) - STRUCT_FOR_ID(_flags) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index c7cd83807b7cb4..6ade8fb6eade03 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -743,7 +743,6 @@ extern "C" { INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ INIT_ID(_fix_up_module), \ - INIT_ID(_flags), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 118fd30adcd94d..0b33ea187e60ff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -564,9 +564,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_fix_up_module); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_flags); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_flags_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string);