From 8942b70712facbc2aaeb15fff2009cd5b599ade7 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 24 Oct 2024 20:23:32 +0100 Subject: [PATCH 01/41] Provide basicsize from PyClassObject instead of directly using size_of This allows for variable sized base classes in future by using negative basicsize values. --- src/pycell/impl_.rs | 11 +++++++++++ src/pyclass/create_type_object.rs | 8 ++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 1b0724d8481..9d08f896cb2 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -294,6 +294,17 @@ impl PyClassObject { self.contents.value.get() } + /// used to set PyType_Spec::basicsize + /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize + pub(crate) fn basicsize() -> ffi::Py_ssize_t { + let size = std::mem::size_of::(); + + // Py_ssize_t may not be equal to isize on all platforms + #[allow(clippy::useless_conversion)] + size.try_into().expect("size should fit in Py_ssize_t") + } + } + /// Gets the offset of the dictionary from the start of the struct in bytes. pub(crate) fn dict_offset() -> ffi::Py_ssize_t { use memoffset::offset_of; diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 8a02baa8ad1..ffa3ac81618 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -47,7 +47,7 @@ where items_iter: PyClassItemsIter, name: &'static str, module: Option<&'static str>, - size_of: usize, + basicsize: ffi::Py_ssize_t, ) -> PyResult { PyTypeBuilder { slots: Vec::new(), @@ -75,7 +75,7 @@ where .offsets(dict_offset, weaklist_offset) .set_is_basetype(is_basetype) .class_items(items_iter) - .build(py, name, module, size_of) + .build(py, name, module, basicsize) } unsafe { @@ -93,7 +93,7 @@ where T::items_iter(), T::NAME, T::MODULE, - std::mem::size_of::>(), + PyClassObject::::basicsize(), ) } } @@ -417,7 +417,7 @@ impl PyTypeBuilder { py: Python<'_>, name: &'static str, module_name: Option<&'static str>, - basicsize: usize, + basicsize: ffi::Py_ssize_t, ) -> PyResult { // `c_ulong` and `c_uint` have the same size // on some platforms (like windows) From dda77a8dedc8c7be21a6856ea0603ad167bd6800 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 24 Oct 2024 21:42:50 +0100 Subject: [PATCH 02/41] Hide PyClassObject details incompatible with variable sizing. --- src/impl_/pyclass.rs | 11 +++++++++-- src/impl_/pymethods.rs | 2 +- src/pycell/impl_.rs | 12 +++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 85cf95a9e11..49745a3901a 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1195,7 +1195,7 @@ pub unsafe trait OffsetCalculator { // Used in generated implementations of OffsetCalculator pub fn class_offset() -> usize { - offset_of!(PyClassObject, contents) + PyClassObject::::contents_offset() } // Used in generated implementations of OffsetCalculator @@ -1578,6 +1578,8 @@ fn pyo3_get_value< #[cfg(test)] #[cfg(feature = "macros")] mod tests { + use crate::pycell::impl_::PyClassObjectContents; + use super::*; #[test] @@ -1606,9 +1608,14 @@ mod tests { Some(PyMethodDefType::StructMember(member)) => { assert_eq!(unsafe { CStr::from_ptr(member.name) }, ffi::c_str!("value")); assert_eq!(member.type_code, ffi::Py_T_OBJECT_EX); + #[repr(C)] + struct ExpectedLayout { + ob_base: ffi::PyObject, + contents: PyClassObjectContents, + } assert_eq!( member.offset, - (memoffset::offset_of!(PyClassObject, contents) + (memoffset::offset_of!(ExpectedLayout, contents) + memoffset::offset_of!(FrozenClass, value)) as ffi::Py_ssize_t ); diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 58d0c93c240..f4945911507 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -319,7 +319,7 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. let _guard = TraverseGuard(class_object); - let instance = &*class_object.contents.value.get(); + let instance = &*class_object.contents().value.get(); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 9d08f896cb2..7ca6039c2e2 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -277,7 +277,7 @@ where #[repr(C)] pub struct PyClassObject { pub(crate) ob_base: ::LayoutAsBase, - pub(crate) contents: PyClassObjectContents, + contents: PyClassObjectContents, } #[repr(C)] @@ -294,6 +294,10 @@ impl PyClassObject { self.contents.value.get() } + pub(crate) fn contents(&self) -> &PyClassObjectContents { + &self.contents + } + /// used to set PyType_Spec::basicsize /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize pub(crate) fn basicsize() -> ffi::Py_ssize_t { @@ -303,6 +307,10 @@ impl PyClassObject { #[allow(clippy::useless_conversion)] size.try_into().expect("size should fit in Py_ssize_t") } + + /// Gets the offset of the contents from the start of the struct in bytes. + pub(crate) fn contents_offset() -> usize { + memoffset::offset_of!(PyClassObject, contents) } /// Gets the offset of the dictionary from the start of the struct in bytes. @@ -328,9 +336,7 @@ impl PyClassObject { #[allow(clippy::useless_conversion)] offset.try_into().expect("offset should fit in Py_ssize_t") } -} -impl PyClassObject { pub(crate) fn borrow_checker(&self) -> &::Checker { T::PyClassMutability::borrow_checker(self) } From b59fe4878bc906bdb60822fec9a476a1385e7d77 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 24 Oct 2024 23:27:36 +0100 Subject: [PATCH 03/41] specify absolute or relative offsets. Variable sized base classes require the use of relative offsets --- pyo3-macros-backend/src/pyclass.rs | 5 +-- pyo3-macros-backend/src/pymethod.rs | 4 +- src/impl_/pyclass.rs | 67 +++++++++++++++++++++++++---- src/pycell/impl_.rs | 18 +++++--- src/pyclass/create_type_object.rs | 42 ++++++++++++------ 5 files changed, 103 insertions(+), 33 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index e44ac890c9c..c361b72d86c 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -2206,7 +2206,7 @@ impl<'a> PyClassImplsBuilder<'a> { let dict_offset = if self.attr.options.dict.is_some() { quote! { - fn dict_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + fn dict_offset() -> ::std::option::Option<#pyo3_path::impl_::pyclass::PyObjectOffset> { ::std::option::Option::Some(#pyo3_path::impl_::pyclass::dict_offset::()) } } @@ -2214,10 +2214,9 @@ impl<'a> PyClassImplsBuilder<'a> { TokenStream::new() }; - // insert space for weak ref let weaklist_offset = if self.attr.options.weakref.is_some() { quote! { - fn weaklist_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + fn weaklist_offset() -> ::std::option::Option<#pyo3_path::impl_::pyclass::PyObjectOffset> { ::std::option::Option::Some(#pyo3_path::impl_::pyclass::weaklist_offset::()) } } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 1254a8d510b..dc26a30a3f5 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -831,8 +831,8 @@ pub fn impl_py_getter_def( struct Offset; unsafe impl #pyo3_path::impl_::pyclass::OffsetCalculator<#cls, #ty> for Offset { - fn offset() -> usize { - #pyo3_path::impl_::pyclass::class_offset::<#cls>() + + fn offset() -> #pyo3_path::impl_::pyclass::PyObjectOffset { + #pyo3_path::impl_::pyclass::subclass_offset::<#cls>() + #pyo3_path::impl_::pyclass::offset_of!(#cls, #field) } } diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 49745a3901a..cbb25363f5d 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -28,13 +28,13 @@ pub use lazy_type_object::LazyTypeObject; /// Gets the offset of the dictionary from the start of the object in bytes. #[inline] -pub fn dict_offset() -> ffi::Py_ssize_t { +pub fn dict_offset() -> PyObjectOffset { PyClassObject::::dict_offset() } /// Gets the offset of the weakref list from the start of the object in bytes. #[inline] -pub fn weaklist_offset() -> ffi::Py_ssize_t { +pub fn weaklist_offset() -> PyObjectOffset { PyClassObject::::weaklist_offset() } @@ -200,13 +200,17 @@ pub trait PyClassImpl: Sized + 'static { fn items_iter() -> PyClassItemsIter; + /// Used to provide the __dictoffset__ slot + /// (equivalent to [tp_dictoffset](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dictoffset)) #[inline] - fn dict_offset() -> Option { + fn dict_offset() -> Option { None } + /// Used to provide the __weaklistoffset__ slot + /// (equivalent to [tp_weaklistoffset](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_weaklistoffset) #[inline] - fn weaklist_offset() -> Option { + fn weaklist_offset() -> Option { None } @@ -1180,6 +1184,40 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( result } +/// Offset of a field within a `PyClassObject`, in bytes. +#[derive(Clone, Copy)] +pub enum PyObjectOffset { + /// An offset relative to the start of the object + Absolute(ffi::Py_ssize_t), + /// An offset relative to the start of the subclass-specific data. Only allowed when basicsize is negative. + /// + Relative(ffi::Py_ssize_t), +} + +impl PyObjectOffset { + pub fn to_value_and_is_relative(&self) -> (ffi::Py_ssize_t, bool) { + match self { + PyObjectOffset::Absolute(offset) => (*offset, false), + PyObjectOffset::Relative(offset) => (*offset, true), + } + } +} + +impl std::ops::Add for PyObjectOffset { + type Output = PyObjectOffset; + + fn add(self, rhs: usize) -> Self::Output { + // Py_ssize_t may not be equal to isize on all platforms + #[allow(clippy::useless_conversion)] + let rhs: ffi::Py_ssize_t = rhs.try_into().expect("offset should fit in Py_ssize_t"); + + match self { + PyObjectOffset::Absolute(offset) => PyObjectOffset::Absolute(offset + rhs), + PyObjectOffset::Relative(offset) => PyObjectOffset::Relative(offset + rhs), + } + } +} + /// Helper trait to locate field within a `#[pyclass]` for a `#[pyo3(get)]`. /// /// Below MSRV 1.77 we can't use `std::mem::offset_of!`, and the replacement in @@ -1190,11 +1228,11 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( /// The trait is unsafe to implement because producing an incorrect offset will lead to UB. pub unsafe trait OffsetCalculator { /// Offset to the field within a `PyClassObject`, in bytes. - fn offset() -> usize; + fn offset() -> PyObjectOffset; } // Used in generated implementations of OffsetCalculator -pub fn class_offset() -> usize { +pub fn subclass_offset() -> PyObjectOffset { PyClassObject::::contents_offset() } @@ -1274,11 +1312,17 @@ impl< pub fn generate(&self, name: &'static CStr, doc: &'static CStr) -> PyMethodDefType { use crate::pyclass::boolean_struct::private::Boolean; if ClassT::Frozen::VALUE { + let (offset, is_relative) = Offset::offset().to_value_and_is_relative(); + let flags = if is_relative { + ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET + } else { + ffi::Py_READONLY + }; PyMethodDefType::StructMember(ffi::PyMemberDef { name: name.as_ptr(), type_code: ffi::Py_T_OBJECT_EX, - offset: Offset::offset() as ffi::Py_ssize_t, - flags: ffi::Py_READONLY, + offset, + flags, doc: doc.as_ptr(), }) } else { @@ -1497,7 +1541,12 @@ where ClassT: PyClass, Offset: OffsetCalculator, { - unsafe { obj.cast::().add(Offset::offset()).cast::() } + match Offset::offset() { + PyObjectOffset::Absolute(offset) => unsafe { + obj.cast::().add(offset as usize).cast::() + }, + PyObjectOffset::Relative(_) => todo!("not yet supported"), + } } #[allow(deprecated)] diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 7ca6039c2e2..4b245588b0a 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -7,7 +7,7 @@ use std::mem::ManuallyDrop; use std::sync::atomic::{AtomicUsize, Ordering}; use crate::impl_::pyclass::{ - PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, + PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, }; use crate::internal::get_slot::TP_FREE; use crate::type_object::{PyLayout, PySizedLayout}; @@ -309,12 +309,16 @@ impl PyClassObject { } /// Gets the offset of the contents from the start of the struct in bytes. - pub(crate) fn contents_offset() -> usize { - memoffset::offset_of!(PyClassObject, contents) + pub(crate) fn contents_offset() -> PyObjectOffset { + let offset = memoffset::offset_of!(PyClassObject, contents); + + // Py_ssize_t may not be equal to isize on all platforms + #[allow(clippy::useless_conversion)] + PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) } /// Gets the offset of the dictionary from the start of the struct in bytes. - pub(crate) fn dict_offset() -> ffi::Py_ssize_t { + pub(crate) fn dict_offset() -> PyObjectOffset { use memoffset::offset_of; let offset = @@ -322,11 +326,11 @@ impl PyClassObject { // Py_ssize_t may not be equal to isize on all platforms #[allow(clippy::useless_conversion)] - offset.try_into().expect("offset should fit in Py_ssize_t") + PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) } /// Gets the offset of the weakref list from the start of the struct in bytes. - pub(crate) fn weaklist_offset() -> ffi::Py_ssize_t { + pub(crate) fn weaklist_offset() -> PyObjectOffset { use memoffset::offset_of; let offset = @@ -334,7 +338,7 @@ impl PyClassObject { // Py_ssize_t may not be equal to isize on all platforms #[allow(clippy::useless_conversion)] - offset.try_into().expect("offset should fit in Py_ssize_t") + PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) } pub(crate) fn borrow_checker(&self) -> &::Checker { diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index ffa3ac81618..18c54366f3b 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -5,7 +5,7 @@ use crate::{ pycell::PyClassObject, pyclass::{ assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, - tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassItemsIter, + tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassItemsIter, PyObjectOffset, }, pymethods::{Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter, _call_clear}, trampoline::trampoline, @@ -41,8 +41,8 @@ where is_mapping: bool, is_sequence: bool, doc: &'static CStr, - dict_offset: Option, - weaklist_offset: Option, + dict_offset: Option, + weaklist_offset: Option, is_basetype: bool, items_iter: PyClassItemsIter, name: &'static str, @@ -120,7 +120,7 @@ struct PyTypeBuilder { has_setitem: bool, has_traverse: bool, has_clear: bool, - dict_offset: Option, + dict_offset: Option, class_flags: c_ulong, // Before Python 3.9, need to patch in buffer methods manually (they don't work in slots) #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] @@ -357,20 +357,26 @@ impl PyTypeBuilder { fn offsets( mut self, - dict_offset: Option, - #[allow(unused_variables)] weaklist_offset: Option, + dict_offset: Option, + #[allow(unused_variables)] weaklist_offset: Option, ) -> Self { self.dict_offset = dict_offset; #[cfg(Py_3_9)] { #[inline(always)] - fn offset_def(name: &'static CStr, offset: ffi::Py_ssize_t) -> ffi::PyMemberDef { + fn offset_def(name: &'static CStr, offset: PyObjectOffset) -> ffi::PyMemberDef { + let (offset, is_relative) = offset.to_value_and_is_relative(); + let flags = if is_relative { + ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET + } else { + ffi::Py_READONLY + }; ffi::PyMemberDef { name: name.as_ptr().cast(), type_code: ffi::Py_T_PYSSIZET, offset, - flags: ffi::Py_READONLY, + flags, doc: std::ptr::null_mut(), } } @@ -400,12 +406,24 @@ impl PyTypeBuilder { (*(*type_object).tp_as_buffer).bf_releasebuffer = builder.buffer_procs.bf_releasebuffer; - if let Some(dict_offset) = dict_offset { - (*type_object).tp_dictoffset = dict_offset; + match dict_offset { + Some(PyObjectOffset::Absolute(offset)) => { + (*type_object).tp_dictoffset = offset; + } + Some(PyObjectOffset::Relative(_)) => { + panic!("relative offsets not supported until python 3.12") + } + _ => {} } - if let Some(weaklist_offset) = weaklist_offset { - (*type_object).tp_weaklistoffset = weaklist_offset; + match weaklist_offset { + Some(PyObjectOffset::Absolute(offset)) => { + (*type_object).tp_weaklistoffset = offset; + } + Some(PyObjectOffset::Relative(_)) => { + panic!("relative offsets not supported until python 3.12") + } + _ => {} } })); } From d9bb2e8619e62e5c84d1a6ef2f5e18fc050c6c4f Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 26 Oct 2024 12:19:34 +0100 Subject: [PATCH 04/41] use trait for accessing object layout --- src/impl_/pyclass.rs | 8 +++-- src/impl_/pymethods.rs | 2 +- src/instance.rs | 1 + src/pycell.rs | 2 +- src/pycell/impl_.rs | 55 ++++++++++++++++++++++++++----- src/pyclass/create_type_object.rs | 1 + 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index cbb25363f5d..5b5c7f55bcc 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -8,7 +8,7 @@ use crate::{ pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, - pycell::PyBorrowError, + pycell::{impl_::InternalPyClassObjectLayout, PyBorrowError}, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, }; @@ -1189,8 +1189,10 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( pub enum PyObjectOffset { /// An offset relative to the start of the object Absolute(ffi::Py_ssize_t), - /// An offset relative to the start of the subclass-specific data. Only allowed when basicsize is negative. + /// An offset relative to the start of the subclass-specific data. + /// Only allowed when basicsize is negative (which is only allowed for python >=3.12). /// + #[cfg(Py_3_12)] Relative(ffi::Py_ssize_t), } @@ -1198,6 +1200,7 @@ impl PyObjectOffset { pub fn to_value_and_is_relative(&self) -> (ffi::Py_ssize_t, bool) { match self { PyObjectOffset::Absolute(offset) => (*offset, false), + #[cfg(Py_3_12)] PyObjectOffset::Relative(offset) => (*offset, true), } } @@ -1213,6 +1216,7 @@ impl std::ops::Add for PyObjectOffset { match self { PyObjectOffset::Absolute(offset) => PyObjectOffset::Absolute(offset + rhs), + #[cfg(Py_3_12)] PyObjectOffset::Relative(offset) => PyObjectOffset::Relative(offset + rhs), } } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index f4945911507..1e652d6e04d 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -4,7 +4,7 @@ use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; use crate::impl_::pycell::{PyClassObject, PyClassObjectLayout}; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; -use crate::pycell::impl_::PyClassBorrowChecker as _; +use crate::pycell::impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker as _}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; diff --git a/src/instance.rs b/src/instance.rs index 99643e12eb1..6dc84bd020a 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2,6 +2,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::impl_::pycell::PyClassObject; use crate::internal_tricks::ptr_from_ref; +use crate::pycell::impl_::InternalPyClassObjectLayout; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; diff --git a/src/pycell.rs b/src/pycell.rs index 51c9f201068..bbf3804ad01 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -208,7 +208,7 @@ use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::{PyClassBorrowChecker, PyClassObjectLayout}; +use impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker, PyClassObjectLayout}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 4b245588b0a..4418c2e4024 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -224,6 +224,28 @@ pub trait PyClassObjectLayout: PyLayout { unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); } +#[doc(hidden)] +pub(crate) trait InternalPyClassObjectLayout: PyLayout { + fn get_ptr(&self) -> *mut T; + + fn contents(&self) -> &PyClassObjectContents; + + /// used to set PyType_Spec::basicsize + /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize + fn basicsize() -> ffi::Py_ssize_t; + + /// Gets the offset of the contents from the start of the struct in bytes. + fn contents_offset() -> PyObjectOffset; + + /// Gets the offset of the dictionary from the start of the struct in bytes. + fn dict_offset() -> PyObjectOffset; + + /// Gets the offset of the weakref list from the start of the struct in bytes. + fn weaklist_offset() -> PyObjectOffset; + + fn borrow_checker(&self) -> &::Checker; +} + impl PyClassObjectLayout for PyClassObjectBase where U: PySizedLayout, @@ -289,18 +311,18 @@ pub(crate) struct PyClassObjectContents { pub(crate) weakref: T::WeakRef, } -impl PyClassObject { - pub(crate) fn get_ptr(&self) -> *mut T { +impl InternalPyClassObjectLayout for PyClassObject { + fn get_ptr(&self) -> *mut T { self.contents.value.get() } - pub(crate) fn contents(&self) -> &PyClassObjectContents { + fn contents(&self) -> &PyClassObjectContents { &self.contents } /// used to set PyType_Spec::basicsize /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize - pub(crate) fn basicsize() -> ffi::Py_ssize_t { + fn basicsize() -> ffi::Py_ssize_t { let size = std::mem::size_of::(); // Py_ssize_t may not be equal to isize on all platforms @@ -309,7 +331,7 @@ impl PyClassObject { } /// Gets the offset of the contents from the start of the struct in bytes. - pub(crate) fn contents_offset() -> PyObjectOffset { + fn contents_offset() -> PyObjectOffset { let offset = memoffset::offset_of!(PyClassObject, contents); // Py_ssize_t may not be equal to isize on all platforms @@ -318,7 +340,7 @@ impl PyClassObject { } /// Gets the offset of the dictionary from the start of the struct in bytes. - pub(crate) fn dict_offset() -> PyObjectOffset { + fn dict_offset() -> PyObjectOffset { use memoffset::offset_of; let offset = @@ -330,7 +352,7 @@ impl PyClassObject { } /// Gets the offset of the weakref list from the start of the struct in bytes. - pub(crate) fn weaklist_offset() -> PyObjectOffset { + fn weaklist_offset() -> PyObjectOffset { use memoffset::offset_of; let offset = @@ -341,7 +363,7 @@ impl PyClassObject { PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) } - pub(crate) fn borrow_checker(&self) -> &::Checker { + fn borrow_checker(&self) -> &::Checker { T::PyClassMutability::borrow_checker(self) } } @@ -425,6 +447,23 @@ mod tests { #[pyclass(crate = "crate", extends = ImmutableChildOfImmutableBase, frozen)] struct ImmutableChildOfImmutableChildOfImmutableBase; + #[pyclass(crate = "crate", subclass)] + struct BaseWithData(#[allow(unused)] u64); + + #[pyclass(crate = "crate", extends = BaseWithData)] + struct ChildWithData(#[allow(unused)] u64); + + #[pyclass(crate = "crate", extends = BaseWithData)] + struct ChildWithoutData; + + #[test] + fn test_inherited_size() { + let base_size = PyClassObject::::basicsize(); + assert!(base_size > 0); // negative indicates variable sized + assert_eq!(base_size, PyClassObject::::basicsize()); + assert!(base_size < PyClassObject::::basicsize()); + } + fn assert_mutable>() {} fn assert_immutable>() {} fn assert_mutable_with_mutable_ancestor< diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 18c54366f3b..5f94dcfdcd5 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -11,6 +11,7 @@ use crate::{ trampoline::trampoline, }, internal_tricks::ptr_from_ref, + pycell::impl_::InternalPyClassObjectLayout, types::{typeobject::PyTypeMethods, PyType}, Py, PyClass, PyResult, PyTypeInfo, Python, }; From 54fb8a9874ca458eb60aaaab82ea710f953db109 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 26 Oct 2024 20:20:32 +0100 Subject: [PATCH 05/41] classes specify their structure rather than hard coding the static layout --- pyo3-macros-backend/src/pyclass.rs | 5 +- src/impl_/pycell.rs | 3 +- src/impl_/pyclass.rs | 19 +++---- src/impl_/pymethods.rs | 11 +++-- src/instance.rs | 10 ++-- src/pycell/impl_.rs | 79 ++++++++++++++++++------------ src/pyclass/create_type_object.rs | 6 +-- src/type_object.rs | 5 ++ src/types/ellipsis.rs | 9 +++- src/types/mod.rs | 2 + src/types/none.rs | 4 ++ src/types/notimplemented.rs | 9 +++- tests/ui/invalid_base_class.stderr | 20 ++++---- 13 files changed, 113 insertions(+), 69 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index c361b72d86c..5e67efce35c 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1812,6 +1812,8 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; + type Layout = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType as #pyo3_path::type_object::PyTypeInfo>::Layout; + #[inline] fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { use #pyo3_path::prelude::PyTypeMethods; @@ -2301,7 +2303,7 @@ impl<'a> PyClassImplsBuilder<'a> { let pyclass_base_type_impl = attr.options.subclass.map(|subclass| { quote_spanned! { subclass.span() => impl #pyo3_path::impl_::pyclass::PyClassBaseType for #cls { - type LayoutAsBase = #pyo3_path::impl_::pycell::PyClassObject; + type LayoutAsBase = ::Layout; type BaseNativeType = ::BaseNativeType; type Initializer = #pyo3_path::pyclass_init::PyClassInitializer; type PyClassMutability = ::PyClassMutability; @@ -2318,6 +2320,7 @@ impl<'a> PyClassImplsBuilder<'a> { const IS_MAPPING: bool = #is_mapping; const IS_SEQUENCE: bool = #is_sequence; + type Layout = <#cls as #pyo3_path::PyTypeInfo>::Layout; type BaseType = #base; type ThreadChecker = #thread_checker; #inventory diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index 93514c7bb29..dff3a64ec86 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,4 +1,5 @@ //! Externally-accessible implementation of pycell pub use crate::pycell::impl_::{ - GetBorrowChecker, PyClassMutability, PyClassObject, PyClassObjectBase, PyClassObjectLayout, + GetBorrowChecker, PyClassMutability, PyClassObjectBase, PyClassObjectLayout, + PyStaticClassObject, }; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 5b5c7f55bcc..ddd0d75ffcf 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -29,13 +29,13 @@ pub use lazy_type_object::LazyTypeObject; /// Gets the offset of the dictionary from the start of the object in bytes. #[inline] pub fn dict_offset() -> PyObjectOffset { - PyClassObject::::dict_offset() + ::Layout::dict_offset() } /// Gets the offset of the weakref list from the start of the object in bytes. #[inline] pub fn weaklist_offset() -> PyObjectOffset { - PyClassObject::::weaklist_offset() + ::Layout::weaklist_offset() } /// Represents the `__dict__` field for `#[pyclass]`. @@ -167,6 +167,8 @@ pub trait PyClassImpl: Sized + 'static { /// #[pyclass(sequence)] const IS_SEQUENCE: bool = false; + type Layout: InternalPyClassObjectLayout; + /// Base class type BaseType: PyTypeInfo + PyClassBaseType; @@ -900,7 +902,7 @@ macro_rules! generate_pyclass_richcompare_slot { } pub use generate_pyclass_richcompare_slot; -use super::{pycell::PyClassObject, pymethods::BoundRef}; +use super::pymethods::BoundRef; /// Implements a freelist. /// @@ -1114,7 +1116,6 @@ impl PyClassThreadChecker for ThreadCheckerImpl { } /// Trait denoting that this class is suitable to be used as a base type for PyClass. - #[cfg_attr( all(diagnostic_namespace, Py_LIMITED_API), diagnostic::on_unimplemented( @@ -1141,7 +1142,7 @@ pub trait PyClassBaseType: Sized { /// Implementation of tp_dealloc for pyclasses without gc pub(crate) unsafe extern "C" fn tp_dealloc(obj: *mut ffi::PyObject) { - crate::impl_::trampoline::dealloc(obj, PyClassObject::::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, ::Layout::tp_dealloc) } /// Implementation of tp_dealloc for pyclasses with gc @@ -1150,7 +1151,7 @@ pub(crate) unsafe extern "C" fn tp_dealloc_with_gc(obj: *mut ffi::Py { ffi::PyObject_GC_UnTrack(obj.cast()); } - crate::impl_::trampoline::dealloc(obj, PyClassObject::::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, ::Layout::tp_dealloc) } pub(crate) unsafe extern "C" fn get_sequence_item_from_mapping( @@ -1184,7 +1185,7 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( result } -/// Offset of a field within a `PyClassObject`, in bytes. +/// Offset of a field within a PyObject in bytes. #[derive(Clone, Copy)] pub enum PyObjectOffset { /// An offset relative to the start of the object @@ -1231,13 +1232,13 @@ impl std::ops::Add for PyObjectOffset { /// /// The trait is unsafe to implement because producing an incorrect offset will lead to UB. pub unsafe trait OffsetCalculator { - /// Offset to the field within a `PyClassObject`, in bytes. + /// Offset to the field within a PyObject fn offset() -> PyObjectOffset; } // Used in generated implementations of OffsetCalculator pub fn subclass_offset() -> PyObjectOffset { - PyClassObject::::contents_offset() + ::Layout::contents_offset() } // Used in generated implementations of OffsetCalculator diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 1e652d6e04d..23d1643fb7c 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -2,7 +2,7 @@ use crate::exceptions::PyStopAsyncIteration; use crate::gil::LockGIL; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; -use crate::impl_::pycell::{PyClassObject, PyClassObjectLayout}; +use crate::impl_::pycell::PyClassObjectLayout; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; use crate::pycell::impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker as _}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; @@ -20,6 +20,7 @@ use std::os::raw::{c_int, c_void}; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr::null_mut; +use super::pyclass::PyClassImpl; use super::trampoline; /// Python 3.8 and up - __ipow__ has modulo argument correctly populated. @@ -301,7 +302,7 @@ where // SAFETY: `slf` is a valid Python object pointer to a class object of type T, and // traversal is running so no mutations can occur. - let class_object: &PyClassObject = &*slf.cast(); + let class_object: &::Layout = &*slf.cast(); let retval = // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so @@ -309,8 +310,8 @@ where if class_object.check_threadsafe().is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread && class_object.borrow_checker().try_borrow().is_ok() { - struct TraverseGuard<'a, T: PyClass>(&'a PyClassObject); - impl Drop for TraverseGuard<'_, T> { + struct TraverseGuard<'a, U: PyClassImpl, V: InternalPyClassObjectLayout>(&'a V, PhantomData); + impl> Drop for TraverseGuard<'_, U, V> { fn drop(&mut self) { self.0.borrow_checker().release_borrow() } @@ -318,7 +319,7 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. - let _guard = TraverseGuard(class_object); + let _guard = TraverseGuard(class_object, PhantomData); let instance = &*class_object.contents().value.get(); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index 6dc84bd020a..79b59be421a 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,6 +1,6 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; -use crate::impl_::pycell::PyClassObject; +use crate::impl_::pyclass::PyClassImpl; use crate::internal_tricks::ptr_from_ref; use crate::pycell::impl_::InternalPyClassObjectLayout; use crate::pycell::{PyBorrowError, PyBorrowMutError}; @@ -463,7 +463,7 @@ where } #[inline] - pub(crate) fn get_class_object(&self) -> &PyClassObject { + pub(crate) fn get_class_object(&self) -> &::Layout { self.1.get_class_object() } } @@ -1296,10 +1296,10 @@ where /// Get a view on the underlying `PyClass` contents. #[inline] - pub(crate) fn get_class_object(&self) -> &PyClassObject { - let class_object = self.as_ptr().cast::>(); + pub(crate) fn get_class_object(&self) -> &::Layout { + let class_object = self.as_ptr().cast::<::Layout>(); // Safety: Bound is known to contain an object which is laid out in memory as a - // PyClassObject. + // ::Layout object unsafe { &*class_object } } } diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 4418c2e4024..140607bb583 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -177,30 +177,30 @@ impl PyClassBorrowChecker for BorrowChecker { pub trait GetBorrowChecker { fn borrow_checker( - class_object: &PyClassObject, + class_object: &T::Layout, ) -> &::Checker; } impl> GetBorrowChecker for MutableClass { - fn borrow_checker(class_object: &PyClassObject) -> &BorrowChecker { - &class_object.contents.borrow_checker + fn borrow_checker(class_object: &T::Layout) -> &BorrowChecker { + &class_object.contents().borrow_checker } } impl> GetBorrowChecker for ImmutableClass { - fn borrow_checker(class_object: &PyClassObject) -> &EmptySlot { - &class_object.contents.borrow_checker + fn borrow_checker(class_object: &T::Layout) -> &EmptySlot { + &class_object.contents().borrow_checker } } impl, M: PyClassMutability> GetBorrowChecker for ExtendsMutableAncestor where - T::BaseType: PyClassImpl + PyClassBaseType>, + T::BaseType: PyClassImpl + PyClassBaseType::Layout>, ::PyClassMutability: PyClassMutability, { - fn borrow_checker(class_object: &PyClassObject) -> &BorrowChecker { - <::PyClassMutability as GetBorrowChecker>::borrow_checker(&class_object.ob_base) + fn borrow_checker(class_object: &T::Layout) -> &BorrowChecker { + <::PyClassMutability as GetBorrowChecker>::borrow_checker(class_object.ob_base()) } } @@ -211,7 +211,7 @@ pub struct PyClassObjectBase { ob_base: T, } -unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} +unsafe impl PyLayout for PyClassObjectBase where U: PyLayout {} #[doc(hidden)] pub trait PyClassObjectLayout: PyLayout { @@ -225,10 +225,13 @@ pub trait PyClassObjectLayout: PyLayout { } #[doc(hidden)] -pub(crate) trait InternalPyClassObjectLayout: PyLayout { +pub trait InternalPyClassObjectLayout: PyClassObjectLayout { fn get_ptr(&self) -> *mut T; fn contents(&self) -> &PyClassObjectContents; + fn contents_mut(&mut self) -> &mut PyClassObjectContents; + + fn ob_base(&self) -> &::LayoutAsBase; /// used to set PyType_Spec::basicsize /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize @@ -248,7 +251,7 @@ pub(crate) trait InternalPyClassObjectLayout: PyLayout { impl PyClassObjectLayout for PyClassObjectBase where - U: PySizedLayout, + U: PyLayout, T: PyTypeInfo, { fn ensure_threadsafe(&self) {} @@ -297,8 +300,8 @@ where /// The layout of a PyClass as a Python object #[repr(C)] -pub struct PyClassObject { - pub(crate) ob_base: ::LayoutAsBase, +pub struct PyStaticClassObject { + ob_base: ::LayoutAsBase, contents: PyClassObjectContents, } @@ -311,15 +314,23 @@ pub(crate) struct PyClassObjectContents { pub(crate) weakref: T::WeakRef, } -impl InternalPyClassObjectLayout for PyClassObject { +impl InternalPyClassObjectLayout for PyStaticClassObject { fn get_ptr(&self) -> *mut T { self.contents.value.get() } + fn ob_base(&self) -> &::LayoutAsBase { + &self.ob_base + } + fn contents(&self) -> &PyClassObjectContents { &self.contents } + fn contents_mut(&mut self) -> &mut PyClassObjectContents { + &mut self.contents + } + /// used to set PyType_Spec::basicsize /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize fn basicsize() -> ffi::Py_ssize_t { @@ -332,7 +343,7 @@ impl InternalPyClassObjectLayout for PyClassObject { /// Gets the offset of the contents from the start of the struct in bytes. fn contents_offset() -> PyObjectOffset { - let offset = memoffset::offset_of!(PyClassObject, contents); + let offset = memoffset::offset_of!(PyStaticClassObject, contents); // Py_ssize_t may not be equal to isize on all platforms #[allow(clippy::useless_conversion)] @@ -343,8 +354,8 @@ impl InternalPyClassObjectLayout for PyClassObject { fn dict_offset() -> PyObjectOffset { use memoffset::offset_of; - let offset = - offset_of!(PyClassObject, contents) + offset_of!(PyClassObjectContents, dict); + let offset = offset_of!(PyStaticClassObject, contents) + + offset_of!(PyClassObjectContents, dict); // Py_ssize_t may not be equal to isize on all platforms #[allow(clippy::useless_conversion)] @@ -355,8 +366,8 @@ impl InternalPyClassObjectLayout for PyClassObject { fn weaklist_offset() -> PyObjectOffset { use memoffset::offset_of; - let offset = - offset_of!(PyClassObject, contents) + offset_of!(PyClassObjectContents, weakref); + let offset = offset_of!(PyStaticClassObject, contents) + + offset_of!(PyClassObjectContents, weakref); // Py_ssize_t may not be equal to isize on all platforms #[allow(clippy::useless_conversion)] @@ -364,14 +375,16 @@ impl InternalPyClassObjectLayout for PyClassObject { } fn borrow_checker(&self) -> &::Checker { - T::PyClassMutability::borrow_checker(self) + // Safety: T::Layout must be PyStaticClassObject + let slf: &T::Layout = unsafe { std::mem::transmute(self) }; + T::PyClassMutability::borrow_checker(slf) } } -unsafe impl PyLayout for PyClassObject {} -impl PySizedLayout for PyClassObject {} +unsafe impl PyLayout for PyStaticClassObject {} +impl PySizedLayout for PyStaticClassObject {} -impl PyClassObjectLayout for PyClassObject +impl PyClassObjectLayout for PyStaticClassObject where ::LayoutAsBase: PyClassObjectLayout, { @@ -387,12 +400,13 @@ where } unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. - let class_object = &mut *(slf.cast::>()); - if class_object.contents.thread_checker.can_drop(py) { - ManuallyDrop::drop(&mut class_object.contents.value); + let class_object = &mut *(slf.cast::()); + let contents = class_object.contents_mut(); + if contents.thread_checker.can_drop(py) { + ManuallyDrop::drop(&mut contents.value); } - class_object.contents.dict.clear_dict(py); - class_object.contents.weakref.clear_weakrefs(slf, py); + contents.dict.clear_dict(py); + contents.weakref.clear_weakrefs(slf, py); ::LayoutAsBase::tp_dealloc(py, slf) } } @@ -458,10 +472,13 @@ mod tests { #[test] fn test_inherited_size() { - let base_size = PyClassObject::::basicsize(); + let base_size = PyStaticClassObject::::basicsize(); assert!(base_size > 0); // negative indicates variable sized - assert_eq!(base_size, PyClassObject::::basicsize()); - assert!(base_size < PyClassObject::::basicsize()); + assert_eq!( + base_size, + PyStaticClassObject::::basicsize() + ); + assert!(base_size < PyStaticClassObject::::basicsize()); } fn assert_mutable>() {} diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 5f94dcfdcd5..ecb7f7dffb0 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -2,10 +2,10 @@ use crate::{ exceptions::PyTypeError, ffi, impl_::{ - pycell::PyClassObject, pyclass::{ assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, - tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassItemsIter, PyObjectOffset, + tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassImpl, PyClassItemsIter, + PyObjectOffset, }, pymethods::{Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter, _call_clear}, trampoline::trampoline, @@ -94,7 +94,7 @@ where T::items_iter(), T::NAME, T::MODULE, - PyClassObject::::basicsize(), + ::Layout::basicsize(), ) } } diff --git a/src/type_object.rs b/src/type_object.rs index b7cad4ab3b2..87f33debcfa 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,6 +1,8 @@ //! Python type object information use crate::ffi_ptr_ext::FfiPtrExt; +use crate::impl_::pyclass::PyClassImpl; +use crate::pycell::impl_::InternalPyClassObjectLayout; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyType}; use crate::{ffi, Bound, Python}; @@ -42,6 +44,9 @@ pub unsafe trait PyTypeInfo: Sized { /// Module name, if any. const MODULE: Option<&'static str>; + /// The type of object layout to use for ancestors or descendents of this type + type Layout: InternalPyClassObjectLayout; + /// Returns the PyTypeObject instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index ee5898c9013..6f0544db999 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -1,6 +1,9 @@ use crate::{ - ffi, ffi_ptr_ext::FfiPtrExt, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyTypeInfo, - Python, + ffi, + ffi_ptr_ext::FfiPtrExt, + impl_::{pycell::PyStaticClassObject, pyclass::PyClassImpl}, + types::any::PyAnyMethods, + Borrowed, Bound, PyAny, PyTypeInfo, Python, }; /// Represents the Python `Ellipsis` object. @@ -32,6 +35,8 @@ unsafe impl PyTypeInfo for PyEllipsis { const MODULE: Option<&'static str> = None; + type Layout = PyStaticClassObject; + fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_Ellipsis()) } } diff --git a/src/types/mod.rs b/src/types/mod.rs index d84f099e773..66a0f67c930 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -158,6 +158,8 @@ macro_rules! pyobject_native_type_info( const NAME: &'static str = stringify!($name); const MODULE: ::std::option::Option<&'static str> = $module; + type Layout = $crate::impl_::pycell::PyStaticClassObject; + #[inline] #[allow(clippy::redundant_closure_call)] fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { diff --git a/src/types/none.rs b/src/types/none.rs index 1ec12d3f5b0..67a3fc28e4d 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -1,4 +1,6 @@ use crate::ffi_ptr_ext::FfiPtrExt; +use crate::impl_::pycell::PyStaticClassObject; +use crate::impl_::pyclass::PyClassImpl; use crate::{ffi, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyObject, PyTypeInfo, Python}; #[allow(deprecated)] use crate::{IntoPy, ToPyObject}; @@ -32,6 +34,8 @@ unsafe impl PyTypeInfo for PyNone { const MODULE: Option<&'static str> = None; + type Layout = PyStaticClassObject; + fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_None()) } } diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index d93ab466d2d..4fbb836d0b5 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -1,6 +1,9 @@ use crate::{ - ffi, ffi_ptr_ext::FfiPtrExt, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyTypeInfo, - Python, + ffi, + ffi_ptr_ext::FfiPtrExt, + impl_::{pycell::PyStaticClassObject, pyclass::PyClassImpl}, + types::any::PyAnyMethods, + Borrowed, Bound, PyAny, PyTypeInfo, Python, }; /// Represents the Python `NotImplemented` object. @@ -35,6 +38,8 @@ unsafe impl PyTypeInfo for PyNotImplemented { const NAME: &'static str = "NotImplementedType"; const MODULE: Option<&'static str> = None; + type Layout = PyStaticClassObject; + fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_NotImplemented()) } } diff --git a/tests/ui/invalid_base_class.stderr b/tests/ui/invalid_base_class.stderr index c40bed9eaa6..a9262564cec 100644 --- a/tests/ui/invalid_base_class.stderr +++ b/tests/ui/invalid_base_class.stderr @@ -1,8 +1,8 @@ error[E0277]: pyclass `PyBool` cannot be subclassed - --> tests/ui/invalid_base_class.rs:4:19 + --> tests/ui/invalid_base_class.rs:4:1 | 4 | #[pyclass(extends=PyBool)] - | ^^^^^^ required for `#[pyclass(extends=PyBool)]` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyBool)]` | = help: the trait `PyClassBaseType` is not implemented for `PyBool` = note: `PyBool` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -16,17 +16,13 @@ error[E0277]: pyclass `PyBool` cannot be subclassed PyBlockingIOError PyBrokenPipeError and $N others -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: pyclass `PyBool` cannot be subclassed - --> tests/ui/invalid_base_class.rs:4:1 + --> tests/ui/invalid_base_class.rs:4:19 | 4 | #[pyclass(extends=PyBool)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyBool)]` + | ^^^^^^ required for `#[pyclass(extends=PyBool)]` | = help: the trait `PyClassBaseType` is not implemented for `PyBool` = note: `PyBool` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -40,4 +36,8 @@ error[E0277]: pyclass `PyBool` cannot be subclassed PyBlockingIOError PyBrokenPipeError and $N others - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` From 043b2aaa8bd874f84563943b5b504c64db729fb6 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 26 Oct 2024 21:41:17 +0100 Subject: [PATCH 06/41] types specify their layouts --- src/exceptions.rs | 3 ++- src/pycell/impl_.rs | 14 +++++++------- src/types/any.rs | 2 ++ src/types/mod.rs | 13 ++++++++----- src/types/typeobject.rs | 8 +++++++- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/exceptions.rs b/src/exceptions.rs index 6f0fa3e674c..915fef6925d 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -124,7 +124,8 @@ macro_rules! import_exception_bound { $crate::pyobject_native_type_info!( $name, $name::type_object_raw, - ::std::option::Option::Some(stringify!($module)) + ::std::option::Option::Some(stringify!($module)), + $crate::impl_::pycell::PyStaticClassObject ); impl $crate::types::DerefToPyAny for $name {} diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 140607bb583..08c87a37cc8 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -298,13 +298,6 @@ where } } -/// The layout of a PyClass as a Python object -#[repr(C)] -pub struct PyStaticClassObject { - ob_base: ::LayoutAsBase, - contents: PyClassObjectContents, -} - #[repr(C)] pub(crate) struct PyClassObjectContents { pub(crate) value: ManuallyDrop>, @@ -314,6 +307,13 @@ pub(crate) struct PyClassObjectContents { pub(crate) weakref: T::WeakRef, } +/// The layout of a PyClass with a known sized base class as a Python object +#[repr(C)] +pub struct PyStaticClassObject { + ob_base: ::LayoutAsBase, + contents: PyClassObjectContents, +} + impl InternalPyClassObjectLayout for PyStaticClassObject { fn get_ptr(&self) -> *mut T { self.contents.value.get() diff --git a/src/types/any.rs b/src/types/any.rs index e620cf6d137..8e689e8cc68 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -3,6 +3,7 @@ use crate::conversion::{AsPyPointer, FromPyObjectBound, IntoPyObject}; use crate::err::{DowncastError, DowncastIntoError, PyErr, PyResult}; use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::ffi_ptr_ext::FfiPtrExt; +use crate::impl_::pycell::PyStaticClassObject; use crate::instance::Bound; use crate::internal::get_slot::TP_DESCR_GET; use crate::internal_tricks::ptr_from_ref; @@ -41,6 +42,7 @@ pyobject_native_type_info!( PyAny, pyobject_native_static_type_object!(ffi::PyBaseObject_Type), Some("builtins"), + PyStaticClassObject, #checkfunction=PyObject_Check ); diff --git a/src/types/mod.rs b/src/types/mod.rs index 66a0f67c930..864568f4140 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -153,12 +153,12 @@ macro_rules! pyobject_native_static_type_object( #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_info( - ($name:ty, $typeobject:expr, $module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $typeobject:expr, $module:expr, $layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { unsafe impl<$($generics,)*> $crate::type_object::PyTypeInfo for $name { const NAME: &'static str = stringify!($name); const MODULE: ::std::option::Option<&'static str> = $module; - type Layout = $crate::impl_::pycell::PyStaticClassObject; + type Layout = $layout; #[inline] #[allow(clippy::redundant_closure_call)] @@ -186,12 +186,15 @@ macro_rules! pyobject_native_type_info( #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_core { - ($name:ty, $typeobject:expr, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $typeobject:expr, #module=$module:expr, #layout=$layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { $crate::pyobject_native_type_named!($name $(;$generics)*); - $crate::pyobject_native_type_info!($name, $typeobject, $module $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_info!($name, $typeobject, $module, $layout $(, #checkfunction=$checkfunction)? $(;$generics)*); + }; + ($name:ty, $typeobject:expr, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!($name, $typeobject, #module=$module, #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); }; ($name:ty, $typeobject:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins") $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins"), #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); }; } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 7a66b7ad0df..5ddb1b5a672 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -18,7 +18,13 @@ use super::PyString; #[repr(transparent)] pub struct PyType(PyAny); -pyobject_native_type_core!(PyType, pyobject_native_static_type_object!(ffi::PyType_Type), #checkfunction=ffi::PyType_Check); +pyobject_native_type_core!( + PyType, + pyobject_native_static_type_object!(ffi::PyType_Type), + #module=::std::option::Option::Some("builtins"), + #layout=crate::impl_::pycell::PyStaticClassObject, + #checkfunction = ffi::PyType_Check +); impl PyType { /// Creates a new type object. From e5987013c2704c84fb908aba9b39d1b0a3cd4c80 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Oct 2024 21:14:53 +0000 Subject: [PATCH 07/41] variable size class layout using PEP-697 --- src/impl_/pycell.rs | 2 +- src/pycell/impl_.rs | 154 ++++++++++++++++++++++++++++++++++---- src/types/typeobject.rs | 71 +++++++++++++++++- tests/test_inheritance.rs | 104 +++++++++++++++++++++++++ 4 files changed, 311 insertions(+), 20 deletions(-) diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index dff3a64ec86..f9ddd8e8389 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,5 +1,5 @@ //! Externally-accessible implementation of pycell pub use crate::pycell::impl_::{ GetBorrowChecker, PyClassMutability, PyClassObjectBase, PyClassObjectLayout, - PyStaticClassObject, + PyStaticClassObject, PyVariableClassObject, PyVariableClassObjectBase, }; diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 08c87a37cc8..3e237f3f482 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -213,6 +213,25 @@ pub struct PyClassObjectBase { unsafe impl PyLayout for PyClassObjectBase where U: PyLayout {} +/// Base layout of PyClassObject. +#[doc(hidden)] +#[repr(C)] +pub struct PyVariableClassObjectBase { + ob_base: ffi::PyVarObject, +} + +unsafe impl PyLayout for PyVariableClassObjectBase {} + +impl PyClassObjectLayout for PyVariableClassObjectBase { + fn ensure_threadsafe(&self) {} + fn check_threadsafe(&self) -> Result<(), PyBorrowError> { + Ok(()) + } + unsafe fn tp_dealloc(_py: Python<'_>, _slf: *mut ffi::PyObject) { + // TODO(matt): + } +} + #[doc(hidden)] pub trait PyClassObjectLayout: PyLayout { fn ensure_threadsafe(&self); @@ -229,6 +248,7 @@ pub trait InternalPyClassObjectLayout: PyClassObjectLayout { fn get_ptr(&self) -> *mut T; fn contents(&self) -> &PyClassObjectContents; + fn contents_mut(&mut self) -> &mut PyClassObjectContents; fn ob_base(&self) -> &::LayoutAsBase; @@ -307,6 +327,16 @@ pub(crate) struct PyClassObjectContents { pub(crate) weakref: T::WeakRef, } +impl PyClassObjectContents { + unsafe fn dealloc(&mut self, py: Python<'_>, py_object: *mut ffi::PyObject) { + if self.thread_checker.can_drop(py) { + ManuallyDrop::drop(&mut self.value); + } + self.dict.clear_dict(py); + self.weakref.clear_weakrefs(py_object, py); + } +} + /// The layout of a PyClass with a known sized base class as a Python object #[repr(C)] pub struct PyStaticClassObject { @@ -343,11 +373,10 @@ impl InternalPyClassObjectLayout for PyStaticClassObject { /// Gets the offset of the contents from the start of the struct in bytes. fn contents_offset() -> PyObjectOffset { - let offset = memoffset::offset_of!(PyStaticClassObject, contents); - - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) + PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( + PyStaticClassObject, + contents + ))) } /// Gets the offset of the dictionary from the start of the struct in bytes. @@ -357,9 +386,7 @@ impl InternalPyClassObjectLayout for PyStaticClassObject { let offset = offset_of!(PyStaticClassObject, contents) + offset_of!(PyClassObjectContents, dict); - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) } /// Gets the offset of the weakref list from the start of the struct in bytes. @@ -369,9 +396,7 @@ impl InternalPyClassObjectLayout for PyStaticClassObject { let offset = offset_of!(PyStaticClassObject, contents) + offset_of!(PyClassObjectContents, weakref); - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - PyObjectOffset::Absolute(offset.try_into().expect("offset should fit in Py_ssize_t")) + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) } fn borrow_checker(&self) -> &::Checker { @@ -401,16 +426,113 @@ where unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. let class_object = &mut *(slf.cast::()); - let contents = class_object.contents_mut(); - if contents.thread_checker.can_drop(py) { - ManuallyDrop::drop(&mut contents.value); + class_object.contents_mut().dealloc(py, slf); + ::LayoutAsBase::tp_dealloc(py, slf) + } +} + +#[repr(C)] +pub struct PyVariableClassObject { + ob_base: ::LayoutAsBase, +} + +impl PyVariableClassObject { + #[cfg(Py_3_12)] + fn get_contents_ptr(&self) -> *mut PyClassObjectContents { + // https://peps.python.org/pep-0697/ + let obj = self as *const PyVariableClassObject as *mut ffi::PyObject; + let type_obj = unsafe { ffi::Py_TYPE(obj) }; + let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; + return pointer as *mut PyClassObjectContents; + } +} + +#[cfg(Py_3_12)] +impl InternalPyClassObjectLayout for PyVariableClassObject { + fn get_ptr(&self) -> *mut T { + self.contents().value.get() + } + + fn ob_base(&self) -> &::LayoutAsBase { + &self.ob_base + } + + fn contents(&self) -> &PyClassObjectContents { + unsafe { (self.get_contents_ptr() as *const PyClassObjectContents).as_ref() } + .expect("should be able to cast PyClassObjectContents pointer") + } + + fn contents_mut(&mut self) -> &mut PyClassObjectContents { + unsafe { self.get_contents_ptr().as_mut() } + .expect("should be able to cast PyClassObjectContents pointer") + } + + /// used to set PyType_Spec::basicsize + /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize + fn basicsize() -> ffi::Py_ssize_t { + let size = std::mem::size_of::>(); + // negative to indicate 'extra' space that cpython will allocate for us + -usize_to_py_ssize(size) + } + + /// Gets the offset of the contents from the start of the struct in bytes. + fn contents_offset() -> PyObjectOffset { + PyObjectOffset::Relative(0) + } + + /// Gets the offset of the dictionary from the start of the struct in bytes. + fn dict_offset() -> PyObjectOffset { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + dict + ))) + } + + /// Gets the offset of the weakref list from the start of the struct in bytes. + fn weaklist_offset() -> PyObjectOffset { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + weakref + ))) + } + + fn borrow_checker(&self) -> &::Checker { + // Safety: T::Layout must be PyStaticClassObject + let slf: &T::Layout = unsafe { std::mem::transmute(self) }; + T::PyClassMutability::borrow_checker(slf) + } +} + +unsafe impl PyLayout for PyVariableClassObject {} + +impl PyClassObjectLayout for PyVariableClassObject +where + ::LayoutAsBase: PyClassObjectLayout, +{ + fn ensure_threadsafe(&self) { + self.contents().thread_checker.ensure(); + self.ob_base.ensure_threadsafe(); + } + fn check_threadsafe(&self) -> Result<(), PyBorrowError> { + if !self.contents().thread_checker.check() { + return Err(PyBorrowError { _private: () }); } - contents.dict.clear_dict(py); - contents.weakref.clear_weakrefs(slf, py); + self.ob_base.check_threadsafe() + } + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + // Safety: Python only calls tp_dealloc when no references to the object remain. + let class_object = &mut *(slf.cast::()); + class_object.contents_mut().dealloc(py, slf); ::LayoutAsBase::tp_dealloc(py, slf) } } +/// Py_ssize_t may not be equal to isize on all platforms +fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { + #[allow(clippy::useless_conversion)] + value.try_into().expect("value should fit in Py_ssize_t") +} + #[cfg(test)] #[cfg(feature = "macros")] mod tests { diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 5ddb1b5a672..e23ecb2e3f0 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,3 +1,4 @@ +use super::{PyDict, PyString}; use crate::err::{self, PyResult}; use crate::instance::Borrowed; #[cfg(not(Py_3_13))] @@ -6,8 +7,6 @@ use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{ffi, Bound, PyAny, PyTypeInfo, Python}; -use super::PyString; - /// Represents a reference to a Python `type` object. /// /// Values of this type are accessed via PyO3's smart pointers, e.g. as @@ -22,10 +21,17 @@ pyobject_native_type_core!( PyType, pyobject_native_static_type_object!(ffi::PyType_Type), #module=::std::option::Option::Some("builtins"), - #layout=crate::impl_::pycell::PyStaticClassObject, + #layout=crate::impl_::pycell::PyVariableClassObject, #checkfunction = ffi::PyType_Check ); +impl crate::impl_::pyclass::PyClassBaseType for PyType { + type LayoutAsBase = crate::impl_::pycell::PyVariableClassObjectBase; + type BaseNativeType = PyType; + type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; + type PyClassMutability = crate::pycell::impl_::ImmutableClass; +} + impl PyType { /// Creates a new type object. #[inline] @@ -40,6 +46,29 @@ impl PyType { Self::new::(py) } + /// Creates a new type object (class). The resulting type/class will inherit the given metaclass `T` + /// + /// Equivalent to calling `type(name, bases, dict, **kwds)` + /// + #[cfg(not(Py_LIMITED_API))] + pub fn new_type<'py, T: PyTypeInfo>( + py: Python<'py>, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, + ) -> PyResult> { + let new_fn = unsafe { + ffi::PyType_Type + .tp_new + .expect("PyType_Type.tp_new should be present") + }; + let raw_type = T::type_object_raw(py); + let raw_args = args.as_ptr(); + let raw_kwargs = kwargs.map(|v| v.as_ptr()).unwrap_or(std::ptr::null_mut()); + let obj_ptr = unsafe { new_fn(raw_type, raw_args, raw_kwargs) }; + let borrowed_obj = unsafe { Borrowed::from_ptr_or_err(py, obj_ptr) }?; + Ok(borrowed_obj.downcast()?.to_owned()) + } + /// Converts the given FFI pointer into `Bound`, to use in safe code. /// /// The function creates a new reference from the given pointer, and returns @@ -396,4 +425,40 @@ class OuterClass: ); }); } + + #[test] + #[cfg(all(not(Py_LIMITED_API), feature = "macros"))] + fn test_new_type() { + use crate::{ + types::{PyDict, PyList, PyString}, + IntoPy, + }; + + Python::with_gil(|py| { + #[allow(non_snake_case)] + let ListType = py.get_type::(); + let name = PyString::new(py, "MyClass"); + let bases = PyTuple::new(py, [ListType]).unwrap(); + let dict = PyDict::new(py); + dict.set_item("foo", 123_i32.into_py(py)).unwrap(); + let args = PyTuple::new(py, [name.as_any(), bases.as_any(), dict.as_any()]).unwrap(); + #[allow(non_snake_case)] + let MyClass = PyType::new_type::(py, &args, None).unwrap(); + + assert_eq!(MyClass.name().unwrap(), "MyClass"); + assert_eq!(MyClass.qualname().unwrap(), "MyClass"); + + crate::py_run!( + py, + MyClass, + r#" + assert type(MyClass) is type + assert MyClass.__bases__ == (list,) + assert issubclass(MyClass, list) + assert MyClass.foo == 123 + assert not hasattr(MyClass, "__module__") + "# + ); + }); + } } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 7190dd49555..ed1f1467550 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -175,6 +175,110 @@ except Exception as e: }); } +#[cfg(Py_3_12)] +mod inheriting_type { + use super::*; + use pyo3::types::{PyDict, PyTuple}; + + #[test] + fn inherit_type() { + use pyo3::types::PyType; + + #[pyclass(subclass, extends=PyType)] + #[derive(Debug)] + struct Metaclass { + counter: u64, + } + + #[pymethods] + impl Metaclass { + #[new] + #[pyo3(signature = (*args, **kwds))] + fn new<'py>( + py: Python<'py>, + args: &Bound<'py, PyTuple>, + kwds: Option<&Bound<'py, PyDict>>, + ) -> PyResult> { + let type_object = PyType::new_type::(py, args, kwds)?; + type_object.setattr("some_var", 123)?; + Ok(type_object) + } + + fn __getitem__(&self, item: u64) -> u64 { + item + 1 + } + + fn increment_counter(&mut self) { + self.counter += 1; + } + + fn get_counter(&self) -> u64 { + self.counter + } + } + + Python::with_gil(|py| { + #[allow(non_snake_case)] + let Metaclass = py.get_type::(); + + // checking base is `type` + py_run!(py, Metaclass, r#"assert Metaclass.__bases__ == (type,)"#); + + // check can be used as a metaclass + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + value = "foo_value" + assert type(Foo) is Metaclass + assert isinstance(Foo, Metaclass) + assert Foo.value == "foo_value" + assert Foo.some_var == 123 + assert Foo[100] == 101 + FooDynamic = Metaclass("FooDynamic", (), {}) + assert FooDynamic.some_var == 123 + assert FooDynamic[100] == 101 + "# + ); + + // can hold data + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + pass + + assert Foo.get_counter() == 0 + Foo.increment_counter() + assert Foo.get_counter() == 1 + "# + ); + + // can be subclassed + py_run!( + py, + Metaclass, + r#" + class Foo(Metaclass): + value = "foo_value" + + class Bar(metaclass=Metaclass): + value = "bar_value" + + assert Bar.get_counter() == 0 + Bar.increment_counter() + assert Bar.get_counter() == 1 + assert Bar.value == "bar_value" + assert Bar.some_var == 123 + assert Bar[100] == 101 + "# + ); + }); + } +} + // Subclassing builtin types is not allowed in the LIMITED API. #[cfg(not(Py_LIMITED_API))] mod inheriting_native_type { From 8395b8da2a087aeb03fb08f0030bc95806f841c7 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 29 Oct 2024 22:10:38 +0000 Subject: [PATCH 08/41] allow definition of `__init__` methods Metaclasses cannot define `__new__` so `__init__` is the only alternative. --- guide/src/class.md | 2 + guide/src/class/protocols.md | 3 ++ pyo3-macros-backend/src/pymethod.rs | 48 ++++++++++++++++++++++- src/impl_/pyclass.rs | 1 + src/impl_/trampoline.rs | 17 ++++++++ src/pyclass/create_type_object.rs | 10 ++++- src/types/typeobject.rs | 61 +---------------------------- tests/test_inheritance.rs | 18 ++++----- tests/ui/abi3_inheritance.stderr | 28 +++++++------ 9 files changed, 102 insertions(+), 86 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 6a80fd7ad9c..a80dcf3e518 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1374,6 +1374,7 @@ impl pyo3::types::DerefToPyAny for MyClass {} unsafe impl pyo3::type_object::PyTypeInfo for MyClass { const NAME: &'static str = "MyClass"; const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None; + type Layout = pyo3::impl_::pycell::PyStaticClassObject; #[inline] fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject { ::lazy_type_object() @@ -1418,6 +1419,7 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { const IS_SUBCLASS: bool = false; const IS_MAPPING: bool = false; const IS_SEQUENCE: bool = false; + type Layout = ::Layout; type BaseType = PyAny; type ThreadChecker = pyo3::impl_::pyclass::SendablePyClass; type PyClassMutability = <::PyClassMutability as pyo3::impl_::pycell::PyClassMutability>::MutableChild; diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 4d553c276eb..5c109bcbb75 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -141,6 +141,9 @@ given signatures should be interpreted as follows: Determines the "truthyness" of an object. + - `__init__(, ...) -> object` - here, any argument list can be defined + as for normal `pymethods` + - `__call__(, ...) -> object` - here, any argument list can be defined as for normal `pymethods` diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index dc26a30a3f5..3b304834539 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -143,6 +143,7 @@ impl PyMethodKind { "__gt__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GT__)), "__ge__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GE__)), // Some tricky protocols which don't fit the pattern of the rest + "__init__" => PyMethodKind::Proto(PyMethodProtoKind::Init), "__call__" => PyMethodKind::Proto(PyMethodProtoKind::Call), "__traverse__" => PyMethodKind::Proto(PyMethodProtoKind::Traverse), "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Clear), @@ -154,6 +155,7 @@ impl PyMethodKind { enum PyMethodProtoKind { Slot(&'static SlotDef), + Init, Call, Traverse, Clear, @@ -212,6 +214,9 @@ pub fn gen_py_method( let slot = slot_def.generate_type_slot(cls, spec, &method.method_name, ctx)?; GeneratedPyMethod::Proto(slot) } + PyMethodProtoKind::Init => { + GeneratedPyMethod::Proto(impl_init_slot(cls, method.spec, ctx)?) + } PyMethodProtoKind::Call => { GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec, ctx)?) } @@ -303,8 +308,11 @@ fn ensure_no_forbidden_protocol_attributes( method_name: &str, ) -> syn::Result<()> { if let Some(signature) = &spec.signature.attribute { - // __call__ is allowed to have a signature, but nothing else is. - if !matches!(proto_kind, PyMethodProtoKind::Call) { + // __call__ and __init__ are allowed to have a signature, but nothing else is. + if !matches!( + proto_kind, + PyMethodProtoKind::Call | PyMethodProtoKind::Init + ) { bail_spanned!(signature.kw.span() => format!("`signature` cannot be used with magic method `{}`", method_name)); } } @@ -394,6 +402,42 @@ pub fn impl_py_method_def_new( }) } +fn impl_init_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { + let Ctx { pyo3_path, .. } = ctx; + + // HACK: __init__ proto slot must always use varargs calling convention, so change the spec. + // Probably indicates there's a refactoring opportunity somewhere. + spec.convention = CallingConvention::Varargs; + + let wrapper_ident = syn::Ident::new("__pymethod___init____", Span::call_site()); + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; + let slot_def = quote! { + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_init, + pfunc: { + unsafe extern "C" fn trampoline( + slf: *mut #pyo3_path::ffi::PyObject, + args: *mut #pyo3_path::ffi::PyObject, + kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> ::std::os::raw::c_int + { + #pyo3_path::impl_::trampoline::initproc( + slf, + args, + kwargs, + #cls::#wrapper_ident + ) + } + trampoline + } as #pyo3_path::ffi::initproc as _ + } + }; + Ok(MethodAndSlotDef { + associated_method, + slot_def, + }) +} + fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { let Ctx { pyo3_path, .. } = ctx; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index ddd0d75ffcf..5af46423aad 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -167,6 +167,7 @@ pub trait PyClassImpl: Sized + 'static { /// #[pyclass(sequence)] const IS_SEQUENCE: bool = false; + /// Description of how this class is laid out in memory type Layout: InternalPyClassObjectLayout; /// Base class diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 7ffad8abdcd..2cd02d91e85 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -122,6 +122,23 @@ trampolines!( pub fn unaryfunc(slf: *mut ffi::PyObject) -> *mut ffi::PyObject; ); +// tp_init should return 0 on success and -1 on error +// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_init +#[inline] +pub unsafe fn initproc( + slf: *mut ffi::PyObject, + args: *mut ffi::PyObject, + kwargs: *mut ffi::PyObject, + f: for<'py> unsafe fn( + Python<'py>, + *mut ffi::PyObject, + *mut ffi::PyObject, + *mut ffi::PyObject, + ) -> PyResult<*mut ffi::PyObject>, +) -> c_int { + trampoline(|py| f(py, slf, args, kwargs).map(|_| 0)) +} + #[cfg(any(not(Py_LIMITED_API), Py_3_11))] trampoline! { pub fn getbufferproc(slf: *mut ffi::PyObject, buf: *mut ffi::Py_buffer, flags: c_int) -> c_int; diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index ecb7f7dffb0..da90b746505 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -446,8 +446,14 @@ impl PyTypeBuilder { unsafe { self.push_slot(ffi::Py_tp_base, self.tp_base) } - if !self.has_new { - // Safety: This is the correct slot type for Py_tp_new + // Safety: self.tp_base must be a valid PyTypeObject + if unsafe { ffi::PyType_IsSubtype(self.tp_base, &raw mut ffi::PyType_Type) } != 0 { + // if the pyclass derives from `type` (is a metaclass) then `tp_new` must not be set. + // Metaclasses that override tp_new are not supported. + // https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass + assert!(!self.has_new, "Metaclasses must not specify __new__"); + } else if !self.has_new { + // Safety: The default constructor is a valid value of tp_new unsafe { self.push_slot(ffi::Py_tp_new, no_constructor_defined as *mut c_void) } } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index e23ecb2e3f0..40c8169a08f 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,4 +1,4 @@ -use super::{PyDict, PyString}; +use super::PyString; use crate::err::{self, PyResult}; use crate::instance::Borrowed; #[cfg(not(Py_3_13))] @@ -46,29 +46,6 @@ impl PyType { Self::new::(py) } - /// Creates a new type object (class). The resulting type/class will inherit the given metaclass `T` - /// - /// Equivalent to calling `type(name, bases, dict, **kwds)` - /// - #[cfg(not(Py_LIMITED_API))] - pub fn new_type<'py, T: PyTypeInfo>( - py: Python<'py>, - args: &Bound<'py, PyTuple>, - kwargs: Option<&Bound<'py, PyDict>>, - ) -> PyResult> { - let new_fn = unsafe { - ffi::PyType_Type - .tp_new - .expect("PyType_Type.tp_new should be present") - }; - let raw_type = T::type_object_raw(py); - let raw_args = args.as_ptr(); - let raw_kwargs = kwargs.map(|v| v.as_ptr()).unwrap_or(std::ptr::null_mut()); - let obj_ptr = unsafe { new_fn(raw_type, raw_args, raw_kwargs) }; - let borrowed_obj = unsafe { Borrowed::from_ptr_or_err(py, obj_ptr) }?; - Ok(borrowed_obj.downcast()?.to_owned()) - } - /// Converts the given FFI pointer into `Bound`, to use in safe code. /// /// The function creates a new reference from the given pointer, and returns @@ -425,40 +402,4 @@ class OuterClass: ); }); } - - #[test] - #[cfg(all(not(Py_LIMITED_API), feature = "macros"))] - fn test_new_type() { - use crate::{ - types::{PyDict, PyList, PyString}, - IntoPy, - }; - - Python::with_gil(|py| { - #[allow(non_snake_case)] - let ListType = py.get_type::(); - let name = PyString::new(py, "MyClass"); - let bases = PyTuple::new(py, [ListType]).unwrap(); - let dict = PyDict::new(py); - dict.set_item("foo", 123_i32.into_py(py)).unwrap(); - let args = PyTuple::new(py, [name.as_any(), bases.as_any(), dict.as_any()]).unwrap(); - #[allow(non_snake_case)] - let MyClass = PyType::new_type::(py, &args, None).unwrap(); - - assert_eq!(MyClass.name().unwrap(), "MyClass"); - assert_eq!(MyClass.qualname().unwrap(), "MyClass"); - - crate::py_run!( - py, - MyClass, - r#" - assert type(MyClass) is type - assert MyClass.__bases__ == (list,) - assert issubclass(MyClass, list) - assert MyClass.foo == 123 - assert not hasattr(MyClass, "__module__") - "# - ); - }); - } } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index ed1f1467550..2b9cda0cb8a 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -192,16 +192,14 @@ mod inheriting_type { #[pymethods] impl Metaclass { - #[new] - #[pyo3(signature = (*args, **kwds))] - fn new<'py>( - py: Python<'py>, - args: &Bound<'py, PyTuple>, - kwds: Option<&Bound<'py, PyDict>>, - ) -> PyResult> { - let type_object = PyType::new_type::(py, args, kwds)?; - type_object.setattr("some_var", 123)?; - Ok(type_object) + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + slf: Bound<'_, Metaclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) -> PyResult<()> { + slf.as_any().setattr("some_var", 123)?; + Ok(()) } fn __getitem__(&self, item: u64) -> u64 { diff --git a/tests/ui/abi3_inheritance.stderr b/tests/ui/abi3_inheritance.stderr index 309b67a633d..1059c1e4b5c 100644 --- a/tests/ui/abi3_inheritance.stderr +++ b/tests/ui/abi3_inheritance.stderr @@ -1,27 +1,31 @@ error[E0277]: pyclass `PyException` cannot be subclassed - --> tests/ui/abi3_inheritance.rs:4:19 + --> tests/ui/abi3_inheritance.rs:4:1 | 4 | #[pyclass(extends=PyException)] - | ^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` | = help: the trait `PyClassBaseType` is not implemented for `PyException` = note: `PyException` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: pyclass `PyException` cannot be subclassed - --> tests/ui/abi3_inheritance.rs:4:1 + --> tests/ui/abi3_inheritance.rs:4:19 | 4 | #[pyclass(extends=PyException)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` + | ^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` | = help: the trait `PyClassBaseType` is not implemented for `PyException` = note: `PyException` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` From 48379535f8d7fab5014e7a645f73cb863dd1e451 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 29 Oct 2024 22:25:37 +0000 Subject: [PATCH 09/41] add metaclass example to the guide --- guide/src/class/metaclass.md | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 guide/src/class/metaclass.md diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md new file mode 100644 index 00000000000..1eac76465ce --- /dev/null +++ b/guide/src/class/metaclass.md @@ -0,0 +1,69 @@ +# Creating a Metaclass +A [metaclass](https://docs.python.org/3/reference/datamodel.html#metaclasses) is a class that derives `type` and can +be used to influence the construction of other classes. + +Some examples of where metaclasses can be used: + +- [`ABCMeta`](https://docs.python.org/3/library/abc.html) for defining abstract classes +- [`EnumType`](https://docs.python.org/3/library/enum.html) for defining enums +- [`NamedTuple`](https://docs.python.org/3/library/typing.html#typing.NamedTuple) for defining tuples with elements + that can be accessed by name in addition to index. +- to implement patterns such as singleton classes +- automatic registration of classes +- ORM +- serialization / deserialization / validation (e.g. [pydantic](https://docs.pydantic.dev/latest/api/base_model/)) + +### Example: A Simple Metaclass + +```rust +#[pyclass(subclass, extends=PyType)] +struct MyMetaclass { + counter: u64, +}; + +#[pymethods] +impl MyMetaclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + slf: Bound<'_, Metaclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) -> PyResult<()> { + slf.as_any().setattr("some_var", 123)?; + Ok(()) + } + + fn __getitem__(&self, item: u64) -> u64 { + item + 1 + } + + fn increment_counter(&mut self) { + self.counter += 1; + } + + fn get_counter(&self) -> u64 { + self.counter + } +} +``` + +Used like so: +```python +class Foo(metaclass=MyMetaclass): + def __init__() -> None: + ... + +assert type(Foo) is MyMetaclass +assert Foo.some_var == 123 +assert Foo[100] == 101 +Foo.increment_counter() +assert Foo.get_counter() == 1 +``` + +In the example above `MyMetaclass` extends `PyType` (making it a metaclass). It does not define `#[new]` as +[this is not supported](https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass). Instead `__init__` is +defined which is called whenever a class is created that uses `MyMetaclass` as its metaclass. +The arguments to `__init__` are the same as the arguments to `type(name, bases, kwds)`. + +When special methods like `__getitem__` are defined for a metaclass they apply to the classes they construct, so +`Foo[123]` calls `MyMetaclass.__getitem__(Foo, 123)`. From 2132c27396102409edd67e9c8377a77de8322864 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 30 Oct 2024 22:39:52 +0000 Subject: [PATCH 10/41] make class initialization not dependent on the static layout --- src/pycell/impl_.rs | 31 +++++++++++++++++++++++++++---- src/pyclass_init.rs | 23 +++++++---------------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 3e237f3f482..98768b3b7ad 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -3,7 +3,8 @@ use std::cell::UnsafeCell; use std::marker::PhantomData; -use std::mem::ManuallyDrop; +use std::mem::{ManuallyDrop, MaybeUninit}; +use std::ptr::addr_of_mut; use std::sync::atomic::{AtomicUsize, Ordering}; use crate::impl_::pyclass::{ @@ -245,6 +246,10 @@ pub trait PyClassObjectLayout: PyLayout { #[doc(hidden)] pub trait InternalPyClassObjectLayout: PyClassObjectLayout { + /// Obtain a pointer to the contents of an uninitialized PyObject of this type + /// Safety: the provided object must have the layout that the implementation is expecting + unsafe fn contents_uninitialised(obj: *mut ffi::PyObject) -> *mut MaybeUninit>; + fn get_ptr(&self) -> *mut T; fn contents(&self) -> &PyClassObjectContents; @@ -253,7 +258,7 @@ pub trait InternalPyClassObjectLayout: PyClassObjectLayout { fn ob_base(&self) -> &::LayoutAsBase; - /// used to set PyType_Spec::basicsize + /// Used to set PyType_Spec::basicsize /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize fn basicsize() -> ffi::Py_ssize_t; @@ -345,6 +350,16 @@ pub struct PyStaticClassObject { } impl InternalPyClassObjectLayout for PyStaticClassObject { + unsafe fn contents_uninitialised(obj: *mut ffi::PyObject) -> *mut MaybeUninit> { + #[repr(C)] + struct PartiallyInitializedClassObject { + _ob_base: ::LayoutAsBase, + contents: MaybeUninit>, + } + let obj: *mut PartiallyInitializedClassObject = obj.cast(); + addr_of_mut!((*obj).contents) + } + fn get_ptr(&self) -> *mut T { self.contents.value.get() } @@ -438,17 +453,25 @@ pub struct PyVariableClassObject { impl PyVariableClassObject { #[cfg(Py_3_12)] - fn get_contents_ptr(&self) -> *mut PyClassObjectContents { + fn get_contents_of_obj(obj: *mut ffi::PyObject) -> *mut PyClassObjectContents { // https://peps.python.org/pep-0697/ - let obj = self as *const PyVariableClassObject as *mut ffi::PyObject; let type_obj = unsafe { ffi::Py_TYPE(obj) }; let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; return pointer as *mut PyClassObjectContents; } + + #[cfg(Py_3_12)] + fn get_contents_ptr(&self) -> *mut PyClassObjectContents { + Self::get_contents_of_obj(self as *const PyVariableClassObject as *mut ffi::PyObject) + } } #[cfg(Py_3_12)] impl InternalPyClassObjectLayout for PyVariableClassObject { + unsafe fn contents_uninitialised(obj: *mut ffi::PyObject) -> * mut MaybeUninit> { + Self::get_contents_of_obj(obj) as *mut MaybeUninit> + } + fn get_ptr(&self) -> *mut T { self.contents().value.get() } diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 6dc6ec12c6b..90768e3382f 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,19 +1,18 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; -use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; +use crate::impl_::pyclass::{ + PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, +}; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; +use crate::pycell::impl_::InternalPyClassObjectLayout; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; use crate::{ ffi::PyTypeObject, pycell::impl_::{PyClassBorrowChecker, PyClassMutability, PyClassObjectContents}, }; -use std::{ - cell::UnsafeCell, - marker::PhantomData, - mem::{ManuallyDrop, MaybeUninit}, -}; +use std::{cell::UnsafeCell, marker::PhantomData, mem::ManuallyDrop}; /// Initializer for our `#[pyclass]` system. /// @@ -165,14 +164,6 @@ impl PyClassInitializer { where T: PyClass, { - /// Layout of a PyClassObject after base new has been called, but the contents have not yet been - /// written. - #[repr(C)] - struct PartiallyInitializedClassObject { - _ob_base: ::LayoutAsBase, - contents: MaybeUninit>, - } - let (init, super_init) = match self.0 { PyClassInitializerImpl::Existing(value) => return Ok(value.into_bound(py)), PyClassInitializerImpl::New { init, super_init } => (init, super_init), @@ -180,9 +171,9 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; - let part_init: *mut PartiallyInitializedClassObject = obj.cast(); + let contents = ::Layout::contents_uninitialised(obj); std::ptr::write( - (*part_init).contents.as_mut_ptr(), + (*contents).as_mut_ptr(), PyClassObjectContents { value: ManuallyDrop::new(UnsafeCell::new(init)), borrow_checker: ::Storage::new(), From 5f14dd93ad7104d455d452a6b279dfa01cd51ccf Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 2 Nov 2024 12:24:48 +0000 Subject: [PATCH 11/41] handle deallocation for variable layout --- src/pycell/impl_.rs | 82 ++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 98768b3b7ad..35c04c48992 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -228,8 +228,8 @@ impl PyClassObjectLayout for PyVariableClassObjectBase { fn check_threadsafe(&self) -> Result<(), PyBorrowError> { Ok(()) } - unsafe fn tp_dealloc(_py: Python<'_>, _slf: *mut ffi::PyObject) { - // TODO(matt): + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + tp_dealloc(py, slf, T::type_object_raw(py)); } } @@ -284,43 +284,45 @@ where Ok(()) } unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - // FIXME: there is potentially subtle issues here if the base is overwritten - // at runtime? To be investigated. - let type_obj = T::type_object(py); - let type_ptr = type_obj.as_type_ptr(); - let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(slf)); - - // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free - if type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { - let tp_free = actual_type - .get_slot(TP_FREE) - .expect("PyBaseObject_Type should have tp_free"); - return tp_free(slf.cast()); - } + tp_dealloc(py, slf, T::type_object_raw(py)); + } +} - // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. - #[cfg(not(Py_LIMITED_API))] - { - // FIXME: should this be using actual_type.tp_dealloc? - if let Some(dealloc) = (*type_ptr).tp_dealloc { - // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which - // assumes the exception is currently GC tracked, so we have to re-track - // before calling the dealloc so that it can safely call Py_GC_UNTRACK. - #[cfg(not(any(Py_3_11, PyPy)))] - if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { - ffi::PyObject_GC_Track(slf.cast()); - } - dealloc(slf); - } else { - (*actual_type.as_type_ptr()) - .tp_free - .expect("type missing tp_free")(slf.cast()); +unsafe fn tp_dealloc(py: Python<'_>, obj: *mut ffi::PyObject, type_ptr: *mut ffi::PyTypeObject) { + // FIXME: there is potentially subtle issues here if the base is overwritten + // at runtime? To be investigated. + let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); + + // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free + if type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { + let tp_free = actual_type + .get_slot(TP_FREE) + .expect("PyBaseObject_Type should have tp_free"); + return tp_free(obj.cast()); + } + + // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. + #[cfg(not(Py_LIMITED_API))] + { + // FIXME: should this be using actual_type.tp_dealloc? + if let Some(dealloc) = (*type_ptr).tp_dealloc { + // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which + // assumes the exception is currently GC tracked, so we have to re-track + // before calling the dealloc so that it can safely call Py_GC_UNTRACK. + #[cfg(not(any(Py_3_11, PyPy)))] + if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { + ffi::PyObject_GC_Track(obj.cast()); } + dealloc(obj); + } else { + (*actual_type.as_type_ptr()) + .tp_free + .expect("type missing tp_free")(obj.cast()); } - - #[cfg(Py_LIMITED_API)] - unreachable!("subclassing native types is not possible with the `abi3` feature"); } + + #[cfg(Py_LIMITED_API)] + unreachable!("subclassing native types is not possible with the `abi3` feature"); } #[repr(C)] @@ -333,6 +335,16 @@ pub(crate) struct PyClassObjectContents { } impl PyClassObjectContents { + pub(crate) fn new(init: T) -> Self { + PyClassObjectContents { + value: ManuallyDrop::new(UnsafeCell::new(init)), + borrow_checker: ::Storage::new(), + thread_checker: T::ThreadChecker::new(), + dict: T::Dict::INIT, + weakref: T::WeakRef::INIT, + } + } + unsafe fn dealloc(&mut self, py: Python<'_>, py_object: *mut ffi::PyObject) { if self.thread_checker.can_drop(py) { ManuallyDrop::drop(&mut self.value); From 71e6198ca4f8ea96eab2310302fb3f5c3d0ebb06 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 2 Nov 2024 12:27:19 +0000 Subject: [PATCH 12/41] populate using Default before init to avoid data being uninitialized --- guide/src/class/metaclass.md | 4 +- pyo3-macros-backend/src/pymethod.rs | 1 + src/impl_/pyclass_init.rs | 7 + src/pycell/impl_.rs | 12 +- src/pyclass/create_type_object.rs | 14 +- src/pyclass_init.rs | 22 +- tests/test_class_init.rs | 353 +++++++++++++++++++++++++++ tests/test_compile_error.rs | 1 + tests/test_inheritance.rs | 73 +++++- tests/ui/init_without_default.rs | 11 + tests/ui/init_without_default.stderr | 16 ++ 11 files changed, 489 insertions(+), 25 deletions(-) create mode 100644 tests/test_class_init.rs create mode 100644 tests/ui/init_without_default.rs create mode 100644 tests/ui/init_without_default.stderr diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md index 1eac76465ce..98bb0af1dd8 100644 --- a/guide/src/class/metaclass.md +++ b/guide/src/class/metaclass.md @@ -17,6 +17,7 @@ Some examples of where metaclasses can be used: ```rust #[pyclass(subclass, extends=PyType)] +#[derive(Default)] struct MyMetaclass { counter: u64, }; @@ -63,7 +64,8 @@ assert Foo.get_counter() == 1 In the example above `MyMetaclass` extends `PyType` (making it a metaclass). It does not define `#[new]` as [this is not supported](https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass). Instead `__init__` is defined which is called whenever a class is created that uses `MyMetaclass` as its metaclass. -The arguments to `__init__` are the same as the arguments to `type(name, bases, kwds)`. +The arguments to `__init__` are the same as the arguments to `type(name, bases, kwds)`. A `Default` impl is required +in order to define `__init__`. When special methods like `__getitem__` are defined for a metaclass they apply to the classes they construct, so `Foo[123]` calls `MyMetaclass.__getitem__(Foo, 123)`. diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 3b304834539..054d0d68e2d 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -421,6 +421,7 @@ fn impl_init_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result ::std::os::raw::c_int { + #pyo3_path::impl_::pyclass_init::initialize_with_default::<#cls>(slf); #pyo3_path::impl_::trampoline::initproc( slf, args, diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 7242b6186d9..1c9e01fdce3 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -1,11 +1,18 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; +use crate::impl_::pyclass::PyClassImpl; use crate::internal::get_slot::TP_ALLOC; +use crate::pycell::impl_::{InternalPyClassObjectLayout, PyClassObjectContents}; use crate::types::PyType; use crate::{ffi, Borrowed, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; use std::marker::PhantomData; +pub unsafe fn initialize_with_default(obj: *mut ffi::PyObject) { + let contents = T::Layout::contents_uninitialised(obj); + (*contents).write(PyClassObjectContents::new(T::default())); +} + /// Initializer for Python types. /// /// This trait is intended to use internally for distinguishing `#[pyclass]` and diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 35c04c48992..df31714d297 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -248,7 +248,9 @@ pub trait PyClassObjectLayout: PyLayout { pub trait InternalPyClassObjectLayout: PyClassObjectLayout { /// Obtain a pointer to the contents of an uninitialized PyObject of this type /// Safety: the provided object must have the layout that the implementation is expecting - unsafe fn contents_uninitialised(obj: *mut ffi::PyObject) -> *mut MaybeUninit>; + unsafe fn contents_uninitialised( + obj: *mut ffi::PyObject, + ) -> *mut MaybeUninit>; fn get_ptr(&self) -> *mut T; @@ -362,7 +364,9 @@ pub struct PyStaticClassObject { } impl InternalPyClassObjectLayout for PyStaticClassObject { - unsafe fn contents_uninitialised(obj: *mut ffi::PyObject) -> *mut MaybeUninit> { + unsafe fn contents_uninitialised( + obj: *mut ffi::PyObject, + ) -> *mut MaybeUninit> { #[repr(C)] struct PartiallyInitializedClassObject { _ob_base: ::LayoutAsBase, @@ -480,7 +484,9 @@ impl PyVariableClassObject { #[cfg(Py_3_12)] impl InternalPyClassObjectLayout for PyVariableClassObject { - unsafe fn contents_uninitialised(obj: *mut ffi::PyObject) -> * mut MaybeUninit> { + unsafe fn contents_uninitialised( + obj: *mut ffi::PyObject, + ) -> *mut MaybeUninit> { Self::get_contents_of_obj(obj) as *mut MaybeUninit> } diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index da90b746505..9da741c3701 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -62,6 +62,7 @@ where is_mapping, is_sequence, has_new: false, + has_init: false, has_dealloc: false, has_getitem: false, has_setitem: false, @@ -116,6 +117,7 @@ struct PyTypeBuilder { is_mapping: bool, is_sequence: bool, has_new: bool, + has_init: bool, has_dealloc: bool, has_getitem: bool, has_setitem: bool, @@ -134,6 +136,7 @@ impl PyTypeBuilder { unsafe fn push_slot(&mut self, slot: c_int, pfunc: *mut T) { match slot { ffi::Py_tp_new => self.has_new = true, + ffi::Py_tp_init => self.has_init = true, ffi::Py_tp_dealloc => self.has_dealloc = true, ffi::Py_mp_subscript => self.has_getitem = true, ffi::Py_mp_ass_subscript => self.has_setitem = true, @@ -447,11 +450,18 @@ impl PyTypeBuilder { unsafe { self.push_slot(ffi::Py_tp_base, self.tp_base) } // Safety: self.tp_base must be a valid PyTypeObject - if unsafe { ffi::PyType_IsSubtype(self.tp_base, &raw mut ffi::PyType_Type) } != 0 { + let is_metaclass = + unsafe { ffi::PyType_IsSubtype(self.tp_base, &raw mut ffi::PyType_Type) } != 0; + if is_metaclass { // if the pyclass derives from `type` (is a metaclass) then `tp_new` must not be set. // Metaclasses that override tp_new are not supported. // https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass - assert!(!self.has_new, "Metaclasses must not specify __new__"); + assert!( + !self.has_new, + "Metaclasses must not specify __new__ (use __init__ instead)" + ); + // To avoid uninitialized memory, __init__ must be defined instead + assert!(self.has_init, "Metaclasses must specify __init__"); } else if !self.has_new { // Safety: The default constructor is a valid value of tp_new unsafe { self.push_slot(ffi::Py_tp_new, no_constructor_defined as *mut c_void) } diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 90768e3382f..df7a2edeb9e 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,18 +1,13 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; -use crate::impl_::pyclass::{ - PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, -}; +use crate::impl_::pyclass::{PyClassBaseType, PyClassImpl}; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; use crate::pycell::impl_::InternalPyClassObjectLayout; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; -use crate::{ - ffi::PyTypeObject, - pycell::impl_::{PyClassBorrowChecker, PyClassMutability, PyClassObjectContents}, -}; -use std::{cell::UnsafeCell, marker::PhantomData, mem::ManuallyDrop}; +use crate::{ffi::PyTypeObject, pycell::impl_::PyClassObjectContents}; +use std::marker::PhantomData; /// Initializer for our `#[pyclass]` system. /// @@ -172,16 +167,7 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; let contents = ::Layout::contents_uninitialised(obj); - std::ptr::write( - (*contents).as_mut_ptr(), - PyClassObjectContents { - value: ManuallyDrop::new(UnsafeCell::new(init)), - borrow_checker: ::Storage::new(), - thread_checker: T::ThreadChecker::new(), - dict: T::Dict::INIT, - weakref: T::WeakRef::INIT, - }, - ); + (*contents).write(PyClassObjectContents::new(init)); // Safety: obj is a valid pointer to an object of type `target_type`, which` is a known // subclass of `T` diff --git a/tests/test_class_init.rs b/tests/test_class_init.rs new file mode 100644 index 00000000000..175c01e5971 --- /dev/null +++ b/tests/test_class_init.rs @@ -0,0 +1,353 @@ +#![cfg(feature = "macros")] + +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{IntoPyDict, PyDict, PyTuple}, +}; + +#[pyclass] +#[derive(Default)] +struct EmptyClassWithInit {} + +#[pymethods] +impl EmptyClassWithInit { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { + EmptyClassWithInit {} + } + + fn __init__(&self) {} +} + +#[test] +fn empty_class_with_init() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + assert!(typeobj + .call((), None) + .unwrap() + .downcast::() + .is_ok()); + + // Calling with arbitrary args or kwargs is not ok + assert!(typeobj.call(("some", "args"), None).is_err()); + assert!(typeobj + .call((), Some(&[("some", "kwarg")].into_py_dict(py).unwrap())) + .is_err()); + }); +} + +#[pyclass] +struct SimpleInit { + pub number: u64, +} + +impl Default for SimpleInit { + fn default() -> Self { + Self { number: 2 } + } +} + +#[pymethods] +impl SimpleInit { + #[new] + fn new() -> SimpleInit { + SimpleInit { number: 1 } + } + + fn __init__(&mut self) { + assert_eq!(self.number, 2); + self.number = 3; + } +} + +#[test] +fn simple_init() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + let obj = typeobj.call((), None).unwrap(); + let obj = obj.downcast::().unwrap(); + assert_eq!(obj.borrow().number, 3); + + // Calling with arbitrary args or kwargs is not ok + assert!(typeobj.call(("some", "args"), None).is_err()); + assert!(typeobj + .call((), Some(&[("some", "kwarg")].into_py_dict(py).unwrap())) + .is_err()); + }); +} + +#[pyclass] +struct InitWithTwoArgs { + data1: i32, + data2: i32, +} + +impl Default for InitWithTwoArgs { + fn default() -> Self { + Self { + data1: 123, + data2: 234, + } + } +} + +#[pymethods] +impl InitWithTwoArgs { + #[new] + fn new(arg1: i32, _arg2: i32) -> Self { + InitWithTwoArgs { + data1: arg1, + data2: 0, + } + } + + fn __init__(&mut self, _arg1: i32, arg2: i32) { + assert_eq!(self.data1, 123); + assert_eq!(self.data2, 234); + self.data2 = arg2; + } +} + +#[test] +fn init_with_two_args() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + let wrp = typeobj + .call((10, 20), None) + .map_err(|e| e.display(py)) + .unwrap(); + let obj = wrp.downcast::().unwrap(); + let obj_ref = obj.borrow(); + assert_eq!(obj_ref.data1, 123); + assert_eq!(obj_ref.data2, 20); + + assert!(typeobj.call(("a", "b", "c"), None).is_err()); + }); +} + +#[pyclass] +#[derive(Default)] +struct InitWithVarArgs { + args: Option, + kwargs: Option, +} + +#[pymethods] +impl InitWithVarArgs { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { + InitWithVarArgs { + args: None, + kwargs: None, + } + } + + #[pyo3(signature = (*args, **kwargs))] + fn __init__(&mut self, args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>) { + self.args = Some(args.to_string()); + self.kwargs = Some(kwargs.map(|kwargs| kwargs.to_string()).unwrap_or_default()); + } +} + +#[test] +fn init_with_var_args() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + let kwargs = [("a", 1), ("b", 42)].into_py_dict(py).unwrap(); + let wrp = typeobj + .call((10, 20), Some(&kwargs)) + .map_err(|e| e.display(py)) + .unwrap(); + let obj = wrp.downcast::().unwrap(); + let obj_ref = obj.borrow(); + assert_eq!(obj_ref.args, Some("(10, 20)".to_owned())); + assert_eq!(obj_ref.kwargs, Some("{'a': 1, 'b': 42}".to_owned())); + }); +} + +#[pyclass(subclass)] +struct SuperClass { + #[pyo3(get)] + rust_new: bool, + #[pyo3(get)] + rust_default: bool, + #[pyo3(get)] + rust_init: bool, +} + +impl Default for SuperClass { + fn default() -> Self { + Self { + rust_new: false, + rust_default: true, + rust_init: false, + } + } +} + +#[pymethods] +impl SuperClass { + #[new] + fn new() -> Self { + SuperClass { + rust_new: true, + rust_default: false, + rust_init: false, + } + } + + fn __init__(&mut self) { + assert!(!self.rust_new); + assert!(self.rust_default); + assert!(!self.rust_init); + self.rust_init = true; + } +} + +#[test] +fn subclass_init() { + Python::with_gil(|py| { + let super_cls = py.get_type::(); + let source = pyo3_ffi::c_str!(pyo3::indoc::indoc!( + r#" + class Class(SuperClass): + pass + c = Class() + assert c.rust_new is False # overridden because __init__ called + assert c.rust_default is True + assert c.rust_init is True + + class Class(SuperClass): + def __init__(self): + self.py_init = True + c = Class() + assert c.rust_new is True # not overridden because __init__ not called + assert c.rust_default is False + assert c.rust_init is False + assert c.py_init is True + + class Class(SuperClass): + def __init__(self): + super().__init__() + self.py_init = True + c = Class() + assert c.rust_new is False # overridden because __init__ called + assert c.rust_default is True + assert c.rust_init is True + assert c.py_init is True + "# + )); + let globals = PyModule::import(py, "__main__").unwrap().dict(); + globals.set_item("SuperClass", super_cls).unwrap(); + py.run(source, Some(&globals), None) + .map_err(|e| e.display(py)) + .unwrap(); + }); +} + +#[pyclass(extends=SuperClass)] +struct SubClass { + #[pyo3(get)] + rust_subclass_new: bool, + #[pyo3(get)] + rust_subclass_default: bool, + #[pyo3(get)] + rust_subclass_init: bool, +} + +impl Default for SubClass { + fn default() -> Self { + Self { + rust_subclass_new: false, + rust_subclass_default: true, + rust_subclass_init: false, + } + } +} + +#[pymethods] +impl SubClass { + #[new] + fn new() -> (Self, SuperClass) { + ( + SubClass { + rust_subclass_new: true, + rust_subclass_default: false, + rust_subclass_init: false, + }, + SuperClass::new(), + ) + } + + fn __init__(&mut self) { + assert!(!self.rust_subclass_new); + assert!(self.rust_subclass_default); + assert!(!self.rust_subclass_init); + self.rust_subclass_init = true; + } +} + +#[test] +fn subclass_pyclass_init() { + Python::with_gil(|py| { + let sub_cls = py.get_type::(); + let source = pyo3_ffi::c_str!(pyo3::indoc::indoc!( + r#" + c = SubClass() + assert c.rust_new is True + assert c.rust_default is False + assert c.rust_init is False + assert c.rust_subclass_new is False # overridden by calling __init__ + assert c.rust_subclass_default is True + assert c.rust_subclass_init is True + "# + )); + let globals = PyModule::import(py, "__main__").unwrap().dict(); + globals.set_item("SubClass", sub_cls).unwrap(); + py.run(source, Some(&globals), None) + .map_err(|e| e.display(py)) + .unwrap(); + }); +} + +#[pyclass] +#[derive(Debug, Default)] +struct InitWithCustomError {} + +struct CustomError; + +impl From for PyErr { + fn from(_error: CustomError) -> PyErr { + PyValueError::new_err("custom error") + } +} + +#[pymethods] +impl InitWithCustomError { + #[new] + fn new(_should_raise: bool) -> InitWithCustomError { + InitWithCustomError {} + } + + fn __init__(&self, should_raise: bool) -> Result<(), CustomError> { + if should_raise { + Err(CustomError) + } else { + Ok(()) + } + } +} + +#[test] +fn init_with_custom_error() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + typeobj.call((false,), None).unwrap(); + let err = typeobj.call((true,), None).unwrap_err(); + assert_eq!(err.to_string(), "ValueError: custom error"); + }); +} diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index b6cf5065371..9b2b761d817 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -46,6 +46,7 @@ fn test_compile_errors() { t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); t.compile_fail("tests/ui/get_set_all.rs"); + t.compile_fail("tests/ui/init_without_default.rs"); t.compile_fail("tests/ui/traverse.rs"); t.compile_fail("tests/ui/invalid_pymodule_in_root.rs"); t.compile_fail("tests/ui/invalid_pymodule_glob.rs"); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 2b9cda0cb8a..dee2795d3b4 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -185,7 +185,7 @@ mod inheriting_type { use pyo3::types::PyType; #[pyclass(subclass, extends=PyType)] - #[derive(Debug)] + #[derive(Debug, Default)] struct Metaclass { counter: u64, } @@ -275,6 +275,77 @@ mod inheriting_type { ); }); } + + #[test] + #[should_panic = "Metaclasses must specify __init__"] + fn inherit_type_missing_init() { + use pyo3::types::PyType; + + #[pyclass(subclass, extends=PyType)] + #[derive(Debug, Default)] + struct Metaclass {} + + #[pymethods] + impl Metaclass {} + + Python::with_gil(|py| { + #[allow(non_snake_case)] + let Metaclass = py.get_type::(); + + // panics when used + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + pass + "# + ); + }); + } + + #[test] + #[should_panic = "Metaclasses must not specify __new__ (use __init__ instead)"] + fn inherit_type_with_new() { + use pyo3::types::PyType; + + #[pyclass(subclass, extends=PyType)] + #[derive(Debug, Default)] + struct Metaclass {} + + #[pymethods] + impl Metaclass { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn new(_args: Bound<'_, PyTuple>, _kwargs: Option>) -> Self { + Metaclass {} + } + + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + _slf: Bound<'_, Metaclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) -> PyResult<()> { + Ok(()) + } + } + + Python::with_gil(|py| { + #[allow(non_snake_case)] + let Metaclass = py.get_type::(); + + // panics when used + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + pass + "# + ); + }); + } } // Subclassing builtin types is not allowed in the LIMITED API. diff --git a/tests/ui/init_without_default.rs b/tests/ui/init_without_default.rs new file mode 100644 index 00000000000..0d69a3e0a4f --- /dev/null +++ b/tests/ui/init_without_default.rs @@ -0,0 +1,11 @@ +use pyo3::prelude::*; + +#[pyclass] +struct MyClass {} + +#[pymethods] +impl MyClass { + fn __init__(&self) {} +} + +fn main() {} diff --git a/tests/ui/init_without_default.stderr b/tests/ui/init_without_default.stderr new file mode 100644 index 00000000000..8c333a3c05f --- /dev/null +++ b/tests/ui/init_without_default.stderr @@ -0,0 +1,16 @@ +error[E0277]: the trait bound `MyClass: Default` is not satisfied + --> tests/ui/init_without_default.rs:7:6 + | +7 | impl MyClass { + | ^^^^^^^ the trait `Default` is not implemented for `MyClass` + | +note: required by a bound in `initialize_with_default` + --> src/impl_/pyclass_init.rs + | + | pub unsafe fn initialize_with_default(obj: *mut ffi::PyObject) { + | ^^^^^^^ required by this bound in `initialize_with_default` +help: consider annotating `MyClass` with `#[derive(Default)]` + | +4 + #[derive(Default)] +5 | struct MyClass {} + | From 37f85096c5c5a3cb0ee0268dc0b146720b1526fb Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 2 Nov 2024 16:13:24 +0000 Subject: [PATCH 13/41] finish implementation of Offset handling --- src/impl_/pyclass.rs | 26 +++++++--- src/impl_/pyclass_init.rs | 14 +++--- src/pycell/impl_.rs | 8 +-- tests/test_variable_sized_class_basics.rs | 60 +++++++++++++++++++++++ 4 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 tests/test_variable_sized_class_basics.rs diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 5af46423aad..0fbde77872c 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -8,7 +8,10 @@ use crate::{ pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, - pycell::{impl_::InternalPyClassObjectLayout, PyBorrowError}, + pycell::{ + impl_::{InternalPyClassObjectLayout, PyClassObjectContents}, + PyBorrowError, + }, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, }; @@ -1187,7 +1190,7 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( } /// Offset of a field within a PyObject in bytes. -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub enum PyObjectOffset { /// An offset relative to the start of the object Absolute(ffi::Py_ssize_t), @@ -1547,12 +1550,19 @@ where ClassT: PyClass, Offset: OffsetCalculator, { - match Offset::offset() { - PyObjectOffset::Absolute(offset) => unsafe { - obj.cast::().add(offset as usize).cast::() - }, - PyObjectOffset::Relative(_) => todo!("not yet supported"), - } + let (base, offset) = match Offset::offset() { + PyObjectOffset::Absolute(offset) => (obj.cast::(), offset), + #[cfg(Py_3_12)] + PyObjectOffset::Relative(offset) => { + let class_ptr = obj.cast::<::Layout>(); + // Safety: the object `obj` must have the layout `ClassT::Layout` + let class_obj = unsafe { &mut *class_ptr }; + let contents = class_obj.contents_mut() as *mut PyClassObjectContents; + (contents.cast::(), offset) + } + }; + // Safety: conditions for pointer addition must be met + unsafe { base.add(offset as usize) }.cast::() } #[allow(deprecated)] diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 1c9e01fdce3..7a3e598176b 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -44,14 +44,16 @@ impl PyObjectInit for PyNativeTypeInitializer { type_object: *mut PyTypeObject, subtype: *mut PyTypeObject, ) -> PyResult<*mut ffi::PyObject> { - // HACK (due to FIXME below): PyBaseObject_Type's tp_new isn't happy with NULL arguments + // HACK (due to FIXME below): PyBaseObject_Type and PyType_Type tp_new aren't happy with NULL arguments let is_base_object = type_object == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); - let subtype_borrowed: Borrowed<'_, '_, PyType> = subtype - .cast::() - .assume_borrowed_unchecked(py) - .downcast_unchecked(); + let is_metaclass = type_object == std::ptr::addr_of_mut!(ffi::PyType_Type); + + if is_base_object || is_metaclass { + let subtype_borrowed: Borrowed<'_, '_, PyType> = subtype + .cast::() + .assume_borrowed_unchecked(py) + .downcast_unchecked(); - if is_base_object { let alloc = subtype_borrowed .get_slot(TP_ALLOC) .unwrap_or(ffi::PyType_GenericAlloc); diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index df31714d297..accdc9e7efd 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -295,11 +295,13 @@ unsafe fn tp_dealloc(py: Python<'_>, obj: *mut ffi::PyObject, type_ptr: *mut ffi // at runtime? To be investigated. let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); - // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free - if type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { + // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free + let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); + let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); + if is_base_object || is_metaclass { let tp_free = actual_type .get_slot(TP_FREE) - .expect("PyBaseObject_Type should have tp_free"); + .expect("base type should have tp_free"); return tp_free(obj.cast()); } diff --git a/tests/test_variable_sized_class_basics.rs b/tests/test_variable_sized_class_basics.rs new file mode 100644 index 00000000000..5b141e8c9ac --- /dev/null +++ b/tests/test_variable_sized_class_basics.rs @@ -0,0 +1,60 @@ +#![cfg(all(Py_3_12, feature = "macros"))] + +use std::any::TypeId; + +use pyo3::impl_::pycell::PyVariableClassObject; +use pyo3::impl_::pyclass::PyClassImpl; +use pyo3::py_run; +use pyo3::types::{PyDict, PyInt, PyTuple}; +use pyo3::{prelude::*, types::PyType}; + +#[path = "../src/tests/common.rs"] +mod common; + +fn uses_variable_layout() -> bool { + TypeId::of::() == TypeId::of::>() +} + +#[pyclass(extends=PyType)] +#[derive(Default)] +struct ClassWithObjectField { + #[pyo3(get, set)] + value: Option, +} + +#[pymethods] +impl ClassWithObjectField { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + _slf: Bound<'_, ClassWithObjectField>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + } +} + +#[test] +fn class_with_object_field() { + Python::with_gil(|py| { + let ty = py.get_type::(); + assert!(uses_variable_layout::()); + py_run!( + py, + ty, + "x = ty('X', (), {}); x.value = 5; assert x.value == 5" + ); + py_run!( + py, + ty, + "x = ty('X', (), {}); x.value = None; assert x.value == None" + ); + + let obj = Bound::new(py, ClassWithObjectField { value: None }).unwrap(); + py_run!(py, obj, "obj.value = 5"); + let obj_ref = obj.borrow(); + let Some(value) = &obj_ref.value else { + panic!("obj_ref.value is None"); + }; + assert_eq!(*value.downcast_bound::(py).unwrap(), 5); + }); +} From 78b2f9bd8f450b6d7c0b4e92dbbf1b68cdde6e9e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 2 Nov 2024 20:12:13 +0000 Subject: [PATCH 14/41] small fixes and tidying --- guide/src/class/metaclass.md | 4 +- guide/src/class/protocols.md | 6 +- src/impl_/coroutine.rs | 2 +- src/impl_/trampoline.rs | 1 + src/pycell/impl_.rs | 11 ++- src/pyclass/create_type_object.rs | 8 +- tests/test_class_basics.rs | 47 +++++++++++ tests/test_inheritance.rs | 92 +++++++++++---------- tests/ui/abi3_nativetype_inheritance.stderr | 28 ++++--- 9 files changed, 129 insertions(+), 70 deletions(-) diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md index 98bb0af1dd8..8e81b581e92 100644 --- a/guide/src/class/metaclass.md +++ b/guide/src/class/metaclass.md @@ -8,7 +8,7 @@ Some examples of where metaclasses can be used: - [`EnumType`](https://docs.python.org/3/library/enum.html) for defining enums - [`NamedTuple`](https://docs.python.org/3/library/typing.html#typing.NamedTuple) for defining tuples with elements that can be accessed by name in addition to index. -- to implement patterns such as singleton classes +- singleton classes - automatic registration of classes - ORM - serialization / deserialization / validation (e.g. [pydantic](https://docs.pydantic.dev/latest/api/base_model/)) @@ -65,7 +65,7 @@ In the example above `MyMetaclass` extends `PyType` (making it a metaclass). It [this is not supported](https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass). Instead `__init__` is defined which is called whenever a class is created that uses `MyMetaclass` as its metaclass. The arguments to `__init__` are the same as the arguments to `type(name, bases, kwds)`. A `Default` impl is required -in order to define `__init__`. +in order to define `__init__`. The data in the struct is initialised to `Default` before `__init__` is called. When special methods like `__getitem__` are defined for a metaclass they apply to the classes they construct, so `Foo[123]` calls `MyMetaclass.__getitem__(Foo, 123)`. diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 5c109bcbb75..e9af71f1a01 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -141,8 +141,10 @@ given signatures should be interpreted as follows: Determines the "truthyness" of an object. - - `__init__(, ...) -> object` - here, any argument list can be defined - as for normal `pymethods` + - `__init__(, ...) -> ()` - the arguments can be defined as for + normal `pymethods`. The pyclass struct must implement `Default`. + If the class defines `__new__` and `__init__` the values set in + `__new__` are overridden by `Default` before `__init__` is called. - `__call__(, ...) -> object` - here, any argument list can be defined as for normal `pymethods` diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index f893a2c2fe9..0ce3a946ffb 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -6,7 +6,7 @@ use std::{ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, - pycell::impl_::PyClassBorrowChecker, + pycell::impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker}, pyclass::boolean_struct::False, types::{PyAnyMethods, PyString}, IntoPyObject, Py, PyAny, PyClass, PyErr, PyResult, Python, diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 2cd02d91e85..f29e6dce395 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -136,6 +136,7 @@ pub unsafe fn initproc( *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject>, ) -> c_int { + // the map() discards the success value of `f` and converts to the success return value for tp_init (0) trampoline(|py| f(py, slf, args, kwargs).map(|_| 0)) } diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index accdc9e7efd..c60c63ed52c 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -12,9 +12,12 @@ use crate::impl_::pyclass::{ }; use crate::internal::get_slot::TP_FREE; use crate::type_object::{PyLayout, PySizedLayout}; -use crate::types::{PyType, PyTypeMethods}; +use crate::types::PyType; use crate::{ffi, PyClass, PyTypeInfo, Python}; +#[cfg(not(Py_LIMITED_API))] +use crate::types::PyTypeMethods; + use super::{PyBorrowError, PyBorrowMutError}; pub trait PyClassMutability { @@ -212,7 +215,7 @@ pub struct PyClassObjectBase { ob_base: T, } -unsafe impl PyLayout for PyClassObjectBase where U: PyLayout {} +unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} /// Base layout of PyClassObject. #[doc(hidden)] @@ -278,7 +281,7 @@ pub trait InternalPyClassObjectLayout: PyClassObjectLayout { impl PyClassObjectLayout for PyClassObjectBase where - U: PyLayout, + U: PySizedLayout, T: PyTypeInfo, { fn ensure_threadsafe(&self) {} @@ -475,7 +478,7 @@ impl PyVariableClassObject { // https://peps.python.org/pep-0697/ let type_obj = unsafe { ffi::Py_TYPE(obj) }; let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; - return pointer as *mut PyClassObjectContents; + pointer as *mut PyClassObjectContents } #[cfg(Py_3_12)] diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 9da741c3701..bcf4dffe2f4 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -414,9 +414,7 @@ impl PyTypeBuilder { Some(PyObjectOffset::Absolute(offset)) => { (*type_object).tp_dictoffset = offset; } - Some(PyObjectOffset::Relative(_)) => { - panic!("relative offsets not supported until python 3.12") - } + // PyObjectOffset::Relative requires >=3.12 _ => {} } @@ -424,9 +422,7 @@ impl PyTypeBuilder { Some(PyObjectOffset::Absolute(offset)) => { (*type_object).tp_weaklistoffset = offset; } - Some(PyObjectOffset::Relative(_)) => { - panic!("relative offsets not supported until python 3.12") - } + // PyObjectOffset::Relative requires >=3.12 _ => {} } })); diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 4a687a89eea..41993f1e941 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -426,6 +426,53 @@ fn test_tuple_struct_class() { }); } +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pyclass] +struct NoDunderDictSupport { + _pad: [u8; 32], +} + +#[test] +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[should_panic(expected = "inst.a = 1")] +fn no_dunder_dict_support_assignment() { + Python::with_gil(|py| { + let inst = Py::new( + py, + NoDunderDictSupport { + _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", + }, + ) + .unwrap(); + // should panic as this class has no __dict__ + py_run!(py, inst, r#"inst.a = 1"#); + }); +} + +#[test] +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +fn no_dunder_dict_support_setattr() { + Python::with_gil(|py| { + let inst = Py::new( + py, + NoDunderDictSupport { + _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", + }, + ) + .unwrap(); + let err = inst + .into_bound(py) + .as_any() + .setattr("a", 1) + .unwrap_err() + .to_string(); + assert_eq!( + &err, + "AttributeError: 'builtins.NoDunderDictSupport' object has no attribute 'a'" + ); + }); +} + #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] #[pyclass(dict, subclass)] struct DunderDictSupport { diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index dee2795d3b4..8483ac69812 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -178,43 +178,50 @@ except Exception as e: #[cfg(Py_3_12)] mod inheriting_type { use super::*; + use pyo3::types::PyType; use pyo3::types::{PyDict, PyTuple}; - #[test] - fn inherit_type() { - use pyo3::types::PyType; + #[pyclass(subclass, extends=PyType)] + #[derive(Debug)] + struct Metaclass { + counter: u64, + } - #[pyclass(subclass, extends=PyType)] - #[derive(Debug, Default)] - struct Metaclass { - counter: u64, + impl Default for Metaclass { + fn default() -> Self { + Self { counter: 999 } } + } - #[pymethods] - impl Metaclass { - #[pyo3(signature = (*_args, **_kwargs))] - fn __init__( - slf: Bound<'_, Metaclass>, - _args: Bound<'_, PyTuple>, - _kwargs: Option>, - ) -> PyResult<()> { - slf.as_any().setattr("some_var", 123)?; - Ok(()) - } + #[pymethods] + impl Metaclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + slf: Bound<'_, Metaclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) -> PyResult<()> { + let mut slf = slf.borrow_mut(); + assert_eq!(slf.counter, 999); + slf.counter = 5; + Ok(()) + } - fn __getitem__(&self, item: u64) -> u64 { - item + 1 - } + fn __getitem__(&self, item: u64) -> u64 { + item + 1 + } - fn increment_counter(&mut self) { - self.counter += 1; - } + fn increment_counter(&mut self) { + self.counter += 1; + } - fn get_counter(&self) -> u64 { - self.counter - } + fn get_counter(&self) -> u64 { + self.counter } + } + #[test] + fn inherit_type() { Python::with_gil(|py| { #[allow(non_snake_case)] let Metaclass = py.get_type::(); @@ -232,10 +239,9 @@ mod inheriting_type { assert type(Foo) is Metaclass assert isinstance(Foo, Metaclass) assert Foo.value == "foo_value" - assert Foo.some_var == 123 assert Foo[100] == 101 FooDynamic = Metaclass("FooDynamic", (), {}) - assert FooDynamic.some_var == 123 + assert type(FooDynamic) is Metaclass assert FooDynamic[100] == 101 "# ); @@ -248,9 +254,9 @@ mod inheriting_type { class Foo(metaclass=Metaclass): pass - assert Foo.get_counter() == 0 + assert Foo.get_counter() == 5 Foo.increment_counter() - assert Foo.get_counter() == 1 + assert Foo.get_counter() == 6 "# ); @@ -262,14 +268,14 @@ mod inheriting_type { class Foo(Metaclass): value = "foo_value" - class Bar(metaclass=Metaclass): + class Bar(metaclass=Foo): value = "bar_value" - assert Bar.get_counter() == 0 + assert isinstance(Bar, Foo) + assert Bar.get_counter() == 5 Bar.increment_counter() - assert Bar.get_counter() == 1 + assert Bar.get_counter() == 6 assert Bar.value == "bar_value" - assert Bar.some_var == 123 assert Bar[100] == 101 "# ); @@ -283,14 +289,14 @@ mod inheriting_type { #[pyclass(subclass, extends=PyType)] #[derive(Debug, Default)] - struct Metaclass {} + struct MetaclassMissingInit {} #[pymethods] - impl Metaclass {} + impl MetaclassMissingInit {} Python::with_gil(|py| { #[allow(non_snake_case)] - let Metaclass = py.get_type::(); + let Metaclass = py.get_type::(); // panics when used py_run!( @@ -311,19 +317,19 @@ mod inheriting_type { #[pyclass(subclass, extends=PyType)] #[derive(Debug, Default)] - struct Metaclass {} + struct MetaclassWithNew {} #[pymethods] - impl Metaclass { + impl MetaclassWithNew { #[new] #[pyo3(signature = (*_args, **_kwargs))] fn new(_args: Bound<'_, PyTuple>, _kwargs: Option>) -> Self { - Metaclass {} + MetaclassWithNew {} } #[pyo3(signature = (*_args, **_kwargs))] fn __init__( - _slf: Bound<'_, Metaclass>, + _slf: Bound<'_, MetaclassWithNew>, _args: Bound<'_, PyTuple>, _kwargs: Option>, ) -> PyResult<()> { @@ -333,7 +339,7 @@ mod inheriting_type { Python::with_gil(|py| { #[allow(non_snake_case)] - let Metaclass = py.get_type::(); + let Metaclass = py.get_type::(); // panics when used py_run!( diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index 872de60b244..a6886a6b906 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -1,27 +1,31 @@ error[E0277]: pyclass `PyDict` cannot be subclassed - --> tests/ui/abi3_nativetype_inheritance.rs:5:19 + --> tests/ui/abi3_nativetype_inheritance.rs:5:1 | 5 | #[pyclass(extends=PyDict)] - | ^^^^^^ required for `#[pyclass(extends=PyDict)]` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyDict)]` | = help: the trait `PyClassBaseType` is not implemented for `PyDict` = note: `PyDict` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: pyclass `PyDict` cannot be subclassed - --> tests/ui/abi3_nativetype_inheritance.rs:5:1 + --> tests/ui/abi3_nativetype_inheritance.rs:5:19 | 5 | #[pyclass(extends=PyDict)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyDict)]` + | ^^^^^^ required for `#[pyclass(extends=PyDict)]` | = help: the trait `PyClassBaseType` is not implemented for `PyDict` = note: `PyDict` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` From eac629e8e52b964860c07f7e1e5539f1a932ae56 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 2 Nov 2024 22:28:22 +0000 Subject: [PATCH 15/41] add news fragment --- newsfragments/4678.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4678.added.md diff --git a/newsfragments/4678.added.md b/newsfragments/4678.added.md new file mode 100644 index 00000000000..8520219a34f --- /dev/null +++ b/newsfragments/4678.added.md @@ -0,0 +1 @@ +Add support for extending variable/unknown sized base classes (eg `type` to create metaclasses) \ No newline at end of file From d0961fed4673e48062131003ce62e09e0432f099 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 2 Nov 2024 22:41:00 +0000 Subject: [PATCH 16/41] small fix to guide --- guide/src/class/metaclass.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md index 8e81b581e92..fb62bb2d479 100644 --- a/guide/src/class/metaclass.md +++ b/guide/src/class/metaclass.md @@ -30,7 +30,7 @@ impl MyMetaclass { _args: Bound<'_, PyTuple>, _kwargs: Option>, ) -> PyResult<()> { - slf.as_any().setattr("some_var", 123)?; + slf.borrow_mut().counter = 5; Ok(()) } From 5499d1087941c8030d5836471ff32f23fbb1e797 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Nov 2024 15:50:59 +0000 Subject: [PATCH 17/41] fix some tests and linting errors --- Cargo.toml | 2 +- guide/src/class/metaclass.md | 3 +- src/impl_/pyclass.rs | 38 +++++++++---------- src/pycell/impl_.rs | 9 +++++ src/pyclass/create_type_object.rs | 38 ++++++++++++------- src/type_object.rs | 6 +-- tests/test_class_basics.rs | 5 +-- tests/test_compile_error.rs | 3 ++ tests/test_inheritance.rs | 10 ++--- tests/ui/invalid_extend_variable_sized.rs | 14 +++++++ tests/ui/invalid_extend_variable_sized.stderr | 15 ++++++++ 11 files changed, 94 insertions(+), 49 deletions(-) create mode 100644 tests/ui/invalid_extend_variable_sized.rs create mode 100644 tests/ui/invalid_extend_variable_sized.stderr diff --git a/Cargo.toml b/Cargo.toml index 9e931ed00b6..f2c7b42c187 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/noxfile.py", "/.github", "/tests/test_compile_error.rs", "/tests/ui"] edition = "2021" -rust-version = "1.63" +rust-version = "1.65" [dependencies] cfg-if = "1.0" diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md index fb62bb2d479..08ab29ce29f 100644 --- a/guide/src/class/metaclass.md +++ b/guide/src/class/metaclass.md @@ -29,9 +29,8 @@ impl MyMetaclass { slf: Bound<'_, Metaclass>, _args: Bound<'_, PyTuple>, _kwargs: Option>, - ) -> PyResult<()> { + ) { slf.borrow_mut().counter = 5; - Ok(()) } fn __getitem__(&self, item: u64) -> u64 { diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 0fbde77872c..6fa027047c8 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1,3 +1,5 @@ +#[cfg(Py_3_12)] +use crate::pycell::impl_::PyClassObjectContents; use crate::{ conversion::IntoPyObject, exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError}, @@ -8,10 +10,7 @@ use crate::{ pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, - pycell::{ - impl_::{InternalPyClassObjectLayout, PyClassObjectContents}, - PyBorrowError, - }, + pycell::{impl_::InternalPyClassObjectLayout, PyBorrowError}, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, }; @@ -1197,20 +1196,9 @@ pub enum PyObjectOffset { /// An offset relative to the start of the subclass-specific data. /// Only allowed when basicsize is negative (which is only allowed for python >=3.12). /// - #[cfg(Py_3_12)] Relative(ffi::Py_ssize_t), } -impl PyObjectOffset { - pub fn to_value_and_is_relative(&self) -> (ffi::Py_ssize_t, bool) { - match self { - PyObjectOffset::Absolute(offset) => (*offset, false), - #[cfg(Py_3_12)] - PyObjectOffset::Relative(offset) => (*offset, true), - } - } -} - impl std::ops::Add for PyObjectOffset { type Output = PyObjectOffset; @@ -1221,7 +1209,6 @@ impl std::ops::Add for PyObjectOffset { match self { PyObjectOffset::Absolute(offset) => PyObjectOffset::Absolute(offset + rhs), - #[cfg(Py_3_12)] PyObjectOffset::Relative(offset) => PyObjectOffset::Relative(offset + rhs), } } @@ -1321,11 +1308,16 @@ impl< pub fn generate(&self, name: &'static CStr, doc: &'static CStr) -> PyMethodDefType { use crate::pyclass::boolean_struct::private::Boolean; if ClassT::Frozen::VALUE { - let (offset, is_relative) = Offset::offset().to_value_and_is_relative(); - let flags = if is_relative { - ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET - } else { - ffi::Py_READONLY + let (offset, flags) = match Offset::offset() { + PyObjectOffset::Absolute(offset) => (offset, ffi::Py_READONLY), + #[cfg(Py_3_12)] + PyObjectOffset::Relative(offset) => { + (offset, ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET) + } + #[cfg(not(Py_3_12))] + PyObjectOffset::Relative(_) => { + panic!("relative offsets not valid before python 3.12"); + } }; PyMethodDefType::StructMember(ffi::PyMemberDef { name: name.as_ptr(), @@ -1560,6 +1552,10 @@ where let contents = class_obj.contents_mut() as *mut PyClassObjectContents; (contents.cast::(), offset) } + #[cfg(not(Py_3_12))] + PyObjectOffset::Relative(_) => { + panic!("relative offsets not valid before python 3.12"); + } }; // Safety: conditions for pointer addition must be met unsafe { base.add(offset as usize) }.cast::() diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index c60c63ed52c..96eafc2348c 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -248,6 +248,14 @@ pub trait PyClassObjectLayout: PyLayout { } #[doc(hidden)] +#[cfg_attr( + all(diagnostic_namespace), + diagnostic::on_unimplemented( + message = "the class layout is not valid", + label = "required for `#[pyclass(extends=...)]`", + note = "the python version being built against influences which layouts are valid", + ) +)] pub trait InternalPyClassObjectLayout: PyClassObjectLayout { /// Obtain a pointer to the contents of an uninitialized PyObject of this type /// Safety: the provided object must have the layout that the implementation is expecting @@ -551,6 +559,7 @@ impl InternalPyClassObjectLayout for PyVariableClassObject unsafe impl PyLayout for PyVariableClassObject {} +#[cfg(Py_3_12)] impl PyClassObjectLayout for PyVariableClassObject where ::LayoutAsBase: PyClassObjectLayout, diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index bcf4dffe2f4..f902f8fffa6 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -19,7 +19,7 @@ use std::{ collections::HashMap, ffi::{CStr, CString}, os::raw::{c_char, c_int, c_ulong, c_void}, - ptr, + ptr::{self, addr_of_mut}, }; pub(crate) struct PyClassTypeObject { @@ -260,7 +260,11 @@ impl PyTypeBuilder { } get_dict = get_dict_impl; - closure = dict_offset as _; + if let PyObjectOffset::Absolute(offset) = dict_offset { + closure = offset as _; + } else { + unreachable!("PyObjectOffset::Relative requires >=3.12"); + } } property_defs.push(ffi::PyGetSetDef { @@ -370,11 +374,16 @@ impl PyTypeBuilder { { #[inline(always)] fn offset_def(name: &'static CStr, offset: PyObjectOffset) -> ffi::PyMemberDef { - let (offset, is_relative) = offset.to_value_and_is_relative(); - let flags = if is_relative { - ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET - } else { - ffi::Py_READONLY + let (offset, flags) = match offset { + PyObjectOffset::Absolute(offset) => (offset, ffi::Py_READONLY), + #[cfg(Py_3_12)] + PyObjectOffset::Relative(offset) => { + (offset, ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET) + } + #[cfg(not(Py_3_12))] + PyObjectOffset::Relative(_) => { + panic!("relative offsets not valid before python 3.12"); + } }; ffi::PyMemberDef { name: name.as_ptr().cast(), @@ -414,16 +423,19 @@ impl PyTypeBuilder { Some(PyObjectOffset::Absolute(offset)) => { (*type_object).tp_dictoffset = offset; } - // PyObjectOffset::Relative requires >=3.12 - _ => {} + Some(PyObjectOffset::Relative(_)) => { + panic!("PyObjectOffset::Relative requires >=3.12") + } + None => {} } - match weaklist_offset { Some(PyObjectOffset::Absolute(offset)) => { (*type_object).tp_weaklistoffset = offset; } - // PyObjectOffset::Relative requires >=3.12 - _ => {} + Some(PyObjectOffset::Relative(_)) => { + panic!("PyObjectOffset::Relative requires >=3.12") + } + None => {} } })); } @@ -447,7 +459,7 @@ impl PyTypeBuilder { // Safety: self.tp_base must be a valid PyTypeObject let is_metaclass = - unsafe { ffi::PyType_IsSubtype(self.tp_base, &raw mut ffi::PyType_Type) } != 0; + unsafe { ffi::PyType_IsSubtype(self.tp_base, addr_of_mut!(ffi::PyType_Type)) } != 0; if is_metaclass { // if the pyclass derives from `type` (is a metaclass) then `tp_new` must not be set. // Metaclasses that override tp_new are not supported. diff --git a/src/type_object.rs b/src/type_object.rs index 87f33debcfa..aaa6431db07 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -2,7 +2,6 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::PyClassImpl; -use crate::pycell::impl_::InternalPyClassObjectLayout; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyType}; use crate::{ffi, Bound, Python}; @@ -44,8 +43,9 @@ pub unsafe trait PyTypeInfo: Sized { /// Module name, if any. const MODULE: Option<&'static str>; - /// The type of object layout to use for ancestors or descendents of this type - type Layout: InternalPyClassObjectLayout; + /// The type of object layout to use for ancestors or descendants of this type. + /// should implement `InternalPyClassObjectLayout` in order to actually use it as a layout. + type Layout; /// Returns the PyTypeObject instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 41993f1e941..92afc2cfd1c 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -466,10 +466,9 @@ fn no_dunder_dict_support_setattr() { .setattr("a", 1) .unwrap_err() .to_string(); - assert_eq!( - &err, + assert!(err.contains( "AttributeError: 'builtins.NoDunderDictSupport' object has no attribute 'a'" - ); + )); }); } diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 9b2b761d817..ff538788427 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -23,6 +23,9 @@ fn test_compile_errors() { t.compile_fail("tests/ui/reject_generics.rs"); t.compile_fail("tests/ui/deprecations.rs"); t.compile_fail("tests/ui/invalid_closure.rs"); + // only possible to extend variable sized types after 3.12 + #[cfg(not(Py_3_12))] + t.compile_fail("tests/ui/invalid_extend_variable_sized.rs"); t.compile_fail("tests/ui/pyclass_send.rs"); t.compile_fail("tests/ui/invalid_argument_attributes.rs"); t.compile_fail("tests/ui/invalid_intopy_derive.rs"); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 8483ac69812..d308cf1be06 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -200,11 +200,10 @@ mod inheriting_type { slf: Bound<'_, Metaclass>, _args: Bound<'_, PyTuple>, _kwargs: Option>, - ) -> PyResult<()> { + ) { let mut slf = slf.borrow_mut(); assert_eq!(slf.counter, 999); slf.counter = 5; - Ok(()) } fn __getitem__(&self, item: u64) -> u64 { @@ -283,7 +282,7 @@ mod inheriting_type { } #[test] - #[should_panic = "Metaclasses must specify __init__"] + #[should_panic(expected = "Metaclasses must specify __init__")] fn inherit_type_missing_init() { use pyo3::types::PyType; @@ -311,7 +310,7 @@ mod inheriting_type { } #[test] - #[should_panic = "Metaclasses must not specify __new__ (use __init__ instead)"] + #[should_panic(expected = "Metaclasses must not specify __new__ (use __init__ instead)")] fn inherit_type_with_new() { use pyo3::types::PyType; @@ -332,8 +331,7 @@ mod inheriting_type { _slf: Bound<'_, MetaclassWithNew>, _args: Bound<'_, PyTuple>, _kwargs: Option>, - ) -> PyResult<()> { - Ok(()) + ) { } } diff --git a/tests/ui/invalid_extend_variable_sized.rs b/tests/ui/invalid_extend_variable_sized.rs new file mode 100644 index 00000000000..1ebd04a1721 --- /dev/null +++ b/tests/ui/invalid_extend_variable_sized.rs @@ -0,0 +1,14 @@ +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyTuple, PyType}; + +#[pyclass(extends=PyType)] +#[derive(Default)] +struct MyClass {} + +#[pymethods] +impl MyClass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__(&mut self, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) {} +} + +fn main() {} diff --git a/tests/ui/invalid_extend_variable_sized.stderr b/tests/ui/invalid_extend_variable_sized.stderr new file mode 100644 index 00000000000..5acd7e732bf --- /dev/null +++ b/tests/ui/invalid_extend_variable_sized.stderr @@ -0,0 +1,15 @@ +error[E0277]: the class layout is not valid + --> tests/ui/invalid_extend_variable_sized.rs:4:1 + | +4 | #[pyclass(extends=PyType)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=...)]` + | + = help: the trait `pyo3::pycell::impl_::InternalPyClassObjectLayout` is not implemented for `PyVariableClassObject` + = note: the python version being built against influences which layouts are valid + = help: the trait `pyo3::pycell::impl_::InternalPyClassObjectLayout` is implemented for `PyStaticClassObject` +note: required by a bound in `PyClassImpl::Layout` + --> src/impl_/pyclass.rs + | + | type Layout: InternalPyClassObjectLayout; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::Layout` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) From ccce96d768adfd9e52fdff66a6131db67b8dff70 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Nov 2024 15:52:03 +0000 Subject: [PATCH 18/41] add instructions to Contributing.md --- Contributing.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Contributing.md b/Contributing.md index 76af08325fb..7ab6cfe27ea 100644 --- a/Contributing.md +++ b/Contributing.md @@ -109,7 +109,23 @@ You can run these checks yourself with `nox`. Use `nox -l` to list the full set `nox -s clippy-all` #### Tests -`nox -s test` or `cargo test` for Rust tests only, `nox -f pytests/noxfile.py -s test` for Python tests only +`nox -s test` or `cargo test` or `cargo nextest` for Rust tests only, `nox -f pytests/noxfile.py -s test` for +Python tests only. + +Configuring the python interpreter version (`Py_*` cfg options) when running `cargo test` is the same as for regular +packages which is explained in [the docs](https://pyo3.rs/v0.22.5/building-and-distribution). +The easiest way to configure the python version is to install with the system package manager or +[pyenv](https://github.com/pyenv/pyenv) then set `PYO3_PYTHON`. +[uv python install](https://docs.astral.sh/uv/concepts/python-versions/) cannot currently be used as it sets some +[incorrect sysconfig values](https://github.com/astral-sh/uv/issues/8429). + +`Py_LIMITED_API` can be controlled with the `abi3` feature of the `pyo3` crate: + +``` +PYO3_PYTHON=/path/to/python cargo nextest run --package pyo3 --features abi3 ... +``` + +use the `PYO3_PRINT_CONFIG=1` to check the identified configuration. #### Check all conditional compilation `nox -s check-feature-powerset` @@ -183,7 +199,11 @@ PyO3 supports all officially supported Python versions, as well as the latest Py PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible. -The minimum Rust version supported will be decided when the release which bumps Python and Rust versions is made. At the time, the minimum Rust version will be set no higher than the lowest Rust version shipped in the current Debian, RHEL and Alpine Linux distributions. +The minimum Rust version supported will be decided when the release which bumps Python and Rust versions is made. +At the time, the minimum Rust version will be set no higher than the lowest Rust version shipped in the current +[Debian](https://packages.debian.org/search?keywords=rustc), +[RHEL](https://docs.redhat.com/en/documentation/red_hat_developer_tools/1) and +[Alpine Linux](https://pkgs.alpinelinux.org/package/edge/main/x86/rust) distributions. CI tests both the most recent stable Rust version and the minimum supported Rust version. Because of Rust's stability guarantees this is sufficient to confirm support for all Rust versions in between. From 426b218b8111bc5155433ae4af786241513eeeca Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Nov 2024 17:27:27 +0000 Subject: [PATCH 19/41] wrap PyClassObjectContents to prevent it from leaking outside pyo3 --- src/impl_/coroutine.rs | 2 +- src/impl_/pycell.rs | 2 +- src/impl_/pyclass.rs | 10 +- src/impl_/pyclass_init.rs | 8 +- src/impl_/pymethods.rs | 10 +- src/instance.rs | 2 +- src/pycell.rs | 2 +- src/pycell/impl_.rs | 96 ++++++++++++------- src/pyclass/create_type_object.rs | 2 +- src/pyclass_init.rs | 6 +- src/type_object.rs | 2 +- tests/ui/invalid_extend_variable_sized.stderr | 8 +- 12 files changed, 92 insertions(+), 58 deletions(-) diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index 0ce3a946ffb..9856ba0b013 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -6,7 +6,7 @@ use std::{ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, - pycell::impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker}, + pycell::impl_::{PyClassBorrowChecker, PyClassObjectLayout}, pyclass::boolean_struct::False, types::{PyAnyMethods, PyString}, IntoPyObject, Py, PyAny, PyClass, PyErr, PyResult, Python, diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index f9ddd8e8389..850ab1492c5 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,5 +1,5 @@ //! Externally-accessible implementation of pycell pub use crate::pycell::impl_::{ - GetBorrowChecker, PyClassMutability, PyClassObjectBase, PyClassObjectLayout, + GetBorrowChecker, PyClassMutability, PyClassObjectBase, PyClassObjectBaseLayout, PyStaticClassObject, PyVariableClassObject, PyVariableClassObjectBase, }; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 6fa027047c8..b96be514568 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -6,11 +6,11 @@ use crate::{ ffi, impl_::{ freelist::FreeList, - pycell::{GetBorrowChecker, PyClassMutability, PyClassObjectLayout}, + pycell::{GetBorrowChecker, PyClassMutability, PyClassObjectBaseLayout}, pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, - pycell::{impl_::InternalPyClassObjectLayout, PyBorrowError}, + pycell::{impl_::PyClassObjectLayout, PyBorrowError}, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, }; @@ -170,7 +170,7 @@ pub trait PyClassImpl: Sized + 'static { const IS_SEQUENCE: bool = false; /// Description of how this class is laid out in memory - type Layout: InternalPyClassObjectLayout; + type Layout: PyClassObjectLayout; /// Base class type BaseType: PyTypeInfo + PyClassBaseType; @@ -1137,7 +1137,7 @@ impl PyClassThreadChecker for ThreadCheckerImpl { ) )] pub trait PyClassBaseType: Sized { - type LayoutAsBase: PyClassObjectLayout; + type LayoutAsBase: PyClassObjectBaseLayout; type BaseNativeType; type Initializer: PyObjectInit; type PyClassMutability: PyClassMutability; @@ -1549,7 +1549,7 @@ where let class_ptr = obj.cast::<::Layout>(); // Safety: the object `obj` must have the layout `ClassT::Layout` let class_obj = unsafe { &mut *class_ptr }; - let contents = class_obj.contents_mut() as *mut PyClassObjectContents; + let contents = (&mut class_obj.contents_mut().0) as *mut PyClassObjectContents; (contents.cast::(), offset) } #[cfg(not(Py_3_12))] diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 7a3e598176b..496bcdb373a 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -2,7 +2,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::PyClassImpl; use crate::internal::get_slot::TP_ALLOC; -use crate::pycell::impl_::{InternalPyClassObjectLayout, PyClassObjectContents}; +use crate::pycell::impl_::{ + PyClassObjectContents, PyClassObjectLayout, WrappedPyClassObjectContents, +}; use crate::types::PyType; use crate::{ffi, Borrowed, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; @@ -10,7 +12,9 @@ use std::marker::PhantomData; pub unsafe fn initialize_with_default(obj: *mut ffi::PyObject) { let contents = T::Layout::contents_uninitialised(obj); - (*contents).write(PyClassObjectContents::new(T::default())); + (*contents).write(WrappedPyClassObjectContents(PyClassObjectContents::new( + T::default(), + ))); } /// Initializer for Python types. diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 23d1643fb7c..37632873f04 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -2,9 +2,9 @@ use crate::exceptions::PyStopAsyncIteration; use crate::gil::LockGIL; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; -use crate::impl_::pycell::PyClassObjectLayout; +use crate::impl_::pycell::PyClassObjectBaseLayout; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; -use crate::pycell::impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker as _}; +use crate::pycell::impl_::{PyClassBorrowChecker as _, PyClassObjectLayout}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; @@ -310,8 +310,8 @@ where if class_object.check_threadsafe().is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread && class_object.borrow_checker().try_borrow().is_ok() { - struct TraverseGuard<'a, U: PyClassImpl, V: InternalPyClassObjectLayout>(&'a V, PhantomData); - impl> Drop for TraverseGuard<'_, U, V> { + struct TraverseGuard<'a, U: PyClassImpl, V: PyClassObjectLayout>(&'a V, PhantomData); + impl> Drop for TraverseGuard<'_, U, V> { fn drop(&mut self) { self.0.borrow_checker().release_borrow() } @@ -320,7 +320,7 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. let _guard = TraverseGuard(class_object, PhantomData); - let instance = &*class_object.contents().value.get(); + let instance = &*class_object.contents().0.value.get(); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index 79b59be421a..093b8b17fcf 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2,7 +2,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::impl_::pyclass::PyClassImpl; use crate::internal_tricks::ptr_from_ref; -use crate::pycell::impl_::InternalPyClassObjectLayout; +use crate::pycell::impl_::PyClassObjectLayout; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; diff --git a/src/pycell.rs b/src/pycell.rs index bbf3804ad01..057b4c2b77f 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -208,7 +208,7 @@ use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::{InternalPyClassObjectLayout, PyClassBorrowChecker, PyClassObjectLayout}; +use impl_::{PyClassBorrowChecker, PyClassObjectBaseLayout, PyClassObjectLayout}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 96eafc2348c..8bf56c05131 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -187,13 +187,13 @@ pub trait GetBorrowChecker { impl> GetBorrowChecker for MutableClass { fn borrow_checker(class_object: &T::Layout) -> &BorrowChecker { - &class_object.contents().borrow_checker + &class_object.contents().0.borrow_checker } } impl> GetBorrowChecker for ImmutableClass { fn borrow_checker(class_object: &T::Layout) -> &EmptySlot { - &class_object.contents().borrow_checker + &class_object.contents().0.borrow_checker } } @@ -226,7 +226,7 @@ pub struct PyVariableClassObjectBase { unsafe impl PyLayout for PyVariableClassObjectBase {} -impl PyClassObjectLayout for PyVariableClassObjectBase { +impl PyClassObjectBaseLayout for PyVariableClassObjectBase { fn ensure_threadsafe(&self) {} fn check_threadsafe(&self) -> Result<(), PyBorrowError> { Ok(()) @@ -237,7 +237,7 @@ impl PyClassObjectLayout for PyVariableClassObjectBase { } #[doc(hidden)] -pub trait PyClassObjectLayout: PyLayout { +pub trait PyClassObjectBaseLayout: PyLayout { fn ensure_threadsafe(&self); fn check_threadsafe(&self) -> Result<(), PyBorrowError>; /// Implementation of tp_dealloc. @@ -247,6 +247,30 @@ pub trait PyClassObjectLayout: PyLayout { unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); } +/// Allow [PyClassObjectLayout] to have public visibility without leaking the structure of [PyClassObjectContents]. +#[doc(hidden)] +#[repr(transparent)] +pub struct WrappedPyClassObjectContents(pub(crate) PyClassObjectContents); + +impl<'a, T: PyClassImpl> From<&'a PyClassObjectContents> + for &'a WrappedPyClassObjectContents +{ + fn from(value: &'a PyClassObjectContents) -> &'a WrappedPyClassObjectContents { + // Safety: Wrapped struct must use repr(transparent) + unsafe { std::mem::transmute(value) } + } +} + +impl<'a, T: PyClassImpl> From<&'a mut PyClassObjectContents> + for &'a mut WrappedPyClassObjectContents +{ + fn from(value: &'a mut PyClassObjectContents) -> &'a mut WrappedPyClassObjectContents { + // Safety: Wrapped struct must use repr(transparent) + unsafe { std::mem::transmute(value) } + } +} + +/// Functionality required for creating and managing the memory associated with a pyclass annotated struct. #[doc(hidden)] #[cfg_attr( all(diagnostic_namespace), @@ -256,18 +280,18 @@ pub trait PyClassObjectLayout: PyLayout { note = "the python version being built against influences which layouts are valid", ) )] -pub trait InternalPyClassObjectLayout: PyClassObjectLayout { +pub trait PyClassObjectLayout: PyClassObjectBaseLayout { /// Obtain a pointer to the contents of an uninitialized PyObject of this type /// Safety: the provided object must have the layout that the implementation is expecting unsafe fn contents_uninitialised( obj: *mut ffi::PyObject, - ) -> *mut MaybeUninit>; + ) -> *mut MaybeUninit>; fn get_ptr(&self) -> *mut T; - fn contents(&self) -> &PyClassObjectContents; + fn contents(&self) -> &WrappedPyClassObjectContents; - fn contents_mut(&mut self) -> &mut PyClassObjectContents; + fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents; fn ob_base(&self) -> &::LayoutAsBase; @@ -287,7 +311,7 @@ pub trait InternalPyClassObjectLayout: PyClassObjectLayout { fn borrow_checker(&self) -> &::Checker; } -impl PyClassObjectLayout for PyClassObjectBase +impl PyClassObjectBaseLayout for PyClassObjectBase where U: PySizedLayout, T: PyTypeInfo, @@ -301,6 +325,10 @@ where } } +/// Implementation of tp_dealloc. +/// # Safety +/// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. +/// - obj must not be used after this call (as it will be freed). unsafe fn tp_dealloc(py: Python<'_>, obj: *mut ffi::PyObject, type_ptr: *mut ffi::PyTypeObject) { // FIXME: there is potentially subtle issues here if the base is overwritten // at runtime? To be investigated. @@ -376,14 +404,14 @@ pub struct PyStaticClassObject { contents: PyClassObjectContents, } -impl InternalPyClassObjectLayout for PyStaticClassObject { +impl PyClassObjectLayout for PyStaticClassObject { unsafe fn contents_uninitialised( obj: *mut ffi::PyObject, - ) -> *mut MaybeUninit> { + ) -> *mut MaybeUninit> { #[repr(C)] struct PartiallyInitializedClassObject { _ob_base: ::LayoutAsBase, - contents: MaybeUninit>, + contents: MaybeUninit>, } let obj: *mut PartiallyInitializedClassObject = obj.cast(); addr_of_mut!((*obj).contents) @@ -397,12 +425,12 @@ impl InternalPyClassObjectLayout for PyStaticClassObject { &self.ob_base } - fn contents(&self) -> &PyClassObjectContents { - &self.contents + fn contents(&self) -> &WrappedPyClassObjectContents { + (&self.contents).into() } - fn contents_mut(&mut self) -> &mut PyClassObjectContents { - &mut self.contents + fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents { + (&mut self.contents).into() } /// used to set PyType_Spec::basicsize @@ -453,9 +481,9 @@ impl InternalPyClassObjectLayout for PyStaticClassObject { unsafe impl PyLayout for PyStaticClassObject {} impl PySizedLayout for PyStaticClassObject {} -impl PyClassObjectLayout for PyStaticClassObject +impl PyClassObjectBaseLayout for PyStaticClassObject where - ::LayoutAsBase: PyClassObjectLayout, + ::LayoutAsBase: PyClassObjectBaseLayout, { fn ensure_threadsafe(&self) { self.contents.thread_checker.ensure(); @@ -470,7 +498,7 @@ where unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. let class_object = &mut *(slf.cast::()); - class_object.contents_mut().dealloc(py, slf); + class_object.contents_mut().0.dealloc(py, slf); ::LayoutAsBase::tp_dealloc(py, slf) } } @@ -482,41 +510,41 @@ pub struct PyVariableClassObject { impl PyVariableClassObject { #[cfg(Py_3_12)] - fn get_contents_of_obj(obj: *mut ffi::PyObject) -> *mut PyClassObjectContents { + fn get_contents_of_obj(obj: *mut ffi::PyObject) -> *mut WrappedPyClassObjectContents { // https://peps.python.org/pep-0697/ let type_obj = unsafe { ffi::Py_TYPE(obj) }; let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; - pointer as *mut PyClassObjectContents + pointer as *mut WrappedPyClassObjectContents } #[cfg(Py_3_12)] - fn get_contents_ptr(&self) -> *mut PyClassObjectContents { + fn get_contents_ptr(&self) -> *mut WrappedPyClassObjectContents { Self::get_contents_of_obj(self as *const PyVariableClassObject as *mut ffi::PyObject) } } #[cfg(Py_3_12)] -impl InternalPyClassObjectLayout for PyVariableClassObject { +impl PyClassObjectLayout for PyVariableClassObject { unsafe fn contents_uninitialised( obj: *mut ffi::PyObject, - ) -> *mut MaybeUninit> { - Self::get_contents_of_obj(obj) as *mut MaybeUninit> + ) -> *mut MaybeUninit> { + Self::get_contents_of_obj(obj) as *mut MaybeUninit> } fn get_ptr(&self) -> *mut T { - self.contents().value.get() + self.contents().0.value.get() } fn ob_base(&self) -> &::LayoutAsBase { &self.ob_base } - fn contents(&self) -> &PyClassObjectContents { - unsafe { (self.get_contents_ptr() as *const PyClassObjectContents).as_ref() } + fn contents(&self) -> &WrappedPyClassObjectContents { + unsafe { self.get_contents_ptr().cast_const().as_ref() } .expect("should be able to cast PyClassObjectContents pointer") } - fn contents_mut(&mut self) -> &mut PyClassObjectContents { + fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents { unsafe { self.get_contents_ptr().as_mut() } .expect("should be able to cast PyClassObjectContents pointer") } @@ -560,16 +588,16 @@ impl InternalPyClassObjectLayout for PyVariableClassObject unsafe impl PyLayout for PyVariableClassObject {} #[cfg(Py_3_12)] -impl PyClassObjectLayout for PyVariableClassObject +impl PyClassObjectBaseLayout for PyVariableClassObject where - ::LayoutAsBase: PyClassObjectLayout, + ::LayoutAsBase: PyClassObjectBaseLayout, { fn ensure_threadsafe(&self) { - self.contents().thread_checker.ensure(); + self.contents().0.thread_checker.ensure(); self.ob_base.ensure_threadsafe(); } fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - if !self.contents().thread_checker.check() { + if !self.contents().0.thread_checker.check() { return Err(PyBorrowError { _private: () }); } self.ob_base.check_threadsafe() @@ -577,7 +605,7 @@ where unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. let class_object = &mut *(slf.cast::()); - class_object.contents_mut().dealloc(py, slf); + class_object.contents_mut().0.dealloc(py, slf); ::LayoutAsBase::tp_dealloc(py, slf) } } diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index f902f8fffa6..fe0bdbfe5ce 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -11,7 +11,7 @@ use crate::{ trampoline::trampoline, }, internal_tricks::ptr_from_ref, - pycell::impl_::InternalPyClassObjectLayout, + pycell::impl_::PyClassObjectLayout, types::{typeobject::PyTypeMethods, PyType}, Py, PyClass, PyResult, PyTypeInfo, Python, }; diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index df7a2edeb9e..baa1df95b99 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -3,7 +3,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::{PyClassBaseType, PyClassImpl}; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; -use crate::pycell::impl_::InternalPyClassObjectLayout; +use crate::pycell::impl_::{PyClassObjectLayout, WrappedPyClassObjectContents}; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; use crate::{ffi::PyTypeObject, pycell::impl_::PyClassObjectContents}; @@ -167,7 +167,9 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; let contents = ::Layout::contents_uninitialised(obj); - (*contents).write(PyClassObjectContents::new(init)); + (*contents).write(WrappedPyClassObjectContents(PyClassObjectContents::new( + init, + ))); // Safety: obj is a valid pointer to an object of type `target_type`, which` is a known // subclass of `T` diff --git a/src/type_object.rs b/src/type_object.rs index aaa6431db07..0aabcf82a1c 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -44,7 +44,7 @@ pub unsafe trait PyTypeInfo: Sized { const MODULE: Option<&'static str>; /// The type of object layout to use for ancestors or descendants of this type. - /// should implement `InternalPyClassObjectLayout` in order to actually use it as a layout. + /// should implement `PyClassObjectLayout` in order to actually use it as a layout. type Layout; /// Returns the PyTypeObject instance for this type. diff --git a/tests/ui/invalid_extend_variable_sized.stderr b/tests/ui/invalid_extend_variable_sized.stderr index 5acd7e732bf..5f25623d5e7 100644 --- a/tests/ui/invalid_extend_variable_sized.stderr +++ b/tests/ui/invalid_extend_variable_sized.stderr @@ -4,12 +4,12 @@ error[E0277]: the class layout is not valid 4 | #[pyclass(extends=PyType)] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=...)]` | - = help: the trait `pyo3::pycell::impl_::InternalPyClassObjectLayout` is not implemented for `PyVariableClassObject` + = help: the trait `pyo3::pycell::impl_::PyClassObjectLayout` is not implemented for `PyVariableClassObject` = note: the python version being built against influences which layouts are valid - = help: the trait `pyo3::pycell::impl_::InternalPyClassObjectLayout` is implemented for `PyStaticClassObject` + = help: the trait `pyo3::pycell::impl_::PyClassObjectLayout` is implemented for `PyStaticClassObject` note: required by a bound in `PyClassImpl::Layout` --> src/impl_/pyclass.rs | - | type Layout: InternalPyClassObjectLayout; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::Layout` + | type Layout: PyClassObjectLayout; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::Layout` = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) From 0d0868162644ac496d7c111a5067c6406ed27ca1 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Nov 2024 18:26:21 +0000 Subject: [PATCH 20/41] improved docs --- src/pycell/impl_.rs | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 8bf56c05131..4fb34f2deee 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -208,7 +208,8 @@ where } } -/// Base layout of PyClassObject. +/// Base layout of PyClassObject with a known sized base type. +/// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. #[doc(hidden)] #[repr(C)] pub struct PyClassObjectBase { @@ -217,7 +218,8 @@ pub struct PyClassObjectBase { unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} -/// Base layout of PyClassObject. +/// Base layout of PyClassObject with an unknown sized base type. +/// Corresponds to [PyVarObject](https://docs.python.org/3/c-api/structures.html#c.PyVarObject) from the C API. #[doc(hidden)] #[repr(C)] pub struct PyVariableClassObjectBase { @@ -236,6 +238,7 @@ impl PyClassObjectBaseLayout for PyVariableClassObjectBase { } } +/// functionality common to all PyObjects regardless of the layout #[doc(hidden)] pub trait PyClassObjectBaseLayout: PyLayout { fn ensure_threadsafe(&self); @@ -281,22 +284,27 @@ impl<'a, T: PyClassImpl> From<&'a mut PyClassObjectContents> ) )] pub trait PyClassObjectLayout: PyClassObjectBaseLayout { - /// Obtain a pointer to the contents of an uninitialized PyObject of this type + /// Obtain a pointer to the contents of an uninitialized PyObject of this type. + /// /// Safety: the provided object must have the layout that the implementation is expecting unsafe fn contents_uninitialised( obj: *mut ffi::PyObject, ) -> *mut MaybeUninit>; - fn get_ptr(&self) -> *mut T; - + /// obtain a reference to the structure that contains the pyclass struct and associated metadata. fn contents(&self) -> &WrappedPyClassObjectContents; + /// obtain a mutable reference to the structure that contains the pyclass struct and associated metadata. fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents; + /// obtain a pointer to the pyclass struct. + fn get_ptr(&self) -> *mut T; + + /// obtain a reference to the data at the start of the PyObject. fn ob_base(&self) -> &::LayoutAsBase; - /// Used to set PyType_Spec::basicsize - /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize + /// Used to set `PyType_Spec::basicsize` + /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) fn basicsize() -> ffi::Py_ssize_t; /// Gets the offset of the contents from the start of the struct in bytes. @@ -397,7 +405,7 @@ impl PyClassObjectContents { } } -/// The layout of a PyClass with a known sized base class as a Python object +/// The layout of a PyClassObject with a known sized base class. #[repr(C)] pub struct PyStaticClassObject { ob_base: ::LayoutAsBase, @@ -417,14 +425,6 @@ impl PyClassObjectLayout for PyStaticClassObject { addr_of_mut!((*obj).contents) } - fn get_ptr(&self) -> *mut T { - self.contents.value.get() - } - - fn ob_base(&self) -> &::LayoutAsBase { - &self.ob_base - } - fn contents(&self) -> &WrappedPyClassObjectContents { (&self.contents).into() } @@ -433,8 +433,14 @@ impl PyClassObjectLayout for PyStaticClassObject { (&mut self.contents).into() } - /// used to set PyType_Spec::basicsize - /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize + fn get_ptr(&self) -> *mut T { + self.contents.value.get() + } + + fn ob_base(&self) -> &::LayoutAsBase { + &self.ob_base + } + fn basicsize() -> ffi::Py_ssize_t { let size = std::mem::size_of::(); @@ -443,7 +449,6 @@ impl PyClassObjectLayout for PyStaticClassObject { size.try_into().expect("size should fit in Py_ssize_t") } - /// Gets the offset of the contents from the start of the struct in bytes. fn contents_offset() -> PyObjectOffset { PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( PyStaticClassObject, @@ -451,7 +456,6 @@ impl PyClassObjectLayout for PyStaticClassObject { ))) } - /// Gets the offset of the dictionary from the start of the struct in bytes. fn dict_offset() -> PyObjectOffset { use memoffset::offset_of; @@ -461,7 +465,6 @@ impl PyClassObjectLayout for PyStaticClassObject { PyObjectOffset::Absolute(usize_to_py_ssize(offset)) } - /// Gets the offset of the weakref list from the start of the struct in bytes. fn weaklist_offset() -> PyObjectOffset { use memoffset::offset_of; @@ -503,6 +506,10 @@ where } } +/// A layout for a PyClassObject with an unknown sized base type. +/// +/// Utilises [PEP-697](https://peps.python.org/pep-0697/) +#[doc(hidden)] #[repr(C)] pub struct PyVariableClassObject { ob_base: ::LayoutAsBase, @@ -549,20 +556,16 @@ impl PyClassObjectLayout for PyVariableClassObject { .expect("should be able to cast PyClassObjectContents pointer") } - /// used to set PyType_Spec::basicsize - /// https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize fn basicsize() -> ffi::Py_ssize_t { let size = std::mem::size_of::>(); // negative to indicate 'extra' space that cpython will allocate for us -usize_to_py_ssize(size) } - /// Gets the offset of the contents from the start of the struct in bytes. fn contents_offset() -> PyObjectOffset { PyObjectOffset::Relative(0) } - /// Gets the offset of the dictionary from the start of the struct in bytes. fn dict_offset() -> PyObjectOffset { PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( PyClassObjectContents, @@ -570,7 +573,6 @@ impl PyClassObjectLayout for PyVariableClassObject { ))) } - /// Gets the offset of the weakref list from the start of the struct in bytes. fn weaklist_offset() -> PyObjectOffset { PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( PyClassObjectContents, From 85f3802b106f2132e28fe9ea9287b103e7385542 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 4 Nov 2024 21:23:33 +0000 Subject: [PATCH 21/41] introduce OPAQUE constant to begin replacing PyClassObjectLayout Since GAT is not yet supported by the MSRV of pyo3, an alternative approach is required. Using a constant may also end up being simpler. --- guide/src/class/metaclass.md | 2 + pyo3-macros-backend/src/pyclass.rs | 1 + src/exceptions.rs | 1 + src/impl_/pyclass_init.rs | 12 ++--- src/impl_/pymethods.rs | 2 +- src/pycell/impl_.rs | 86 +++++++++++++++++++++--------- src/pyclass_init.rs | 12 ++--- src/type_object.rs | 9 +++- src/types/any.rs | 1 + src/types/ellipsis.rs | 2 +- src/types/mod.rs | 11 ++-- src/types/none.rs | 2 +- src/types/notimplemented.rs | 1 + src/types/typeobject.rs | 1 + 14 files changed, 94 insertions(+), 49 deletions(-) diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md index 08ab29ce29f..b7843e77c36 100644 --- a/guide/src/class/metaclass.md +++ b/guide/src/class/metaclass.md @@ -15,6 +15,8 @@ Some examples of where metaclasses can be used: ### Example: A Simple Metaclass +Note: Creating metaclasses is only possible with python 3.12+ + ```rust #[pyclass(subclass, extends=PyType)] #[derive(Default)] diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 5e67efce35c..e0a9f083a07 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1811,6 +1811,7 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; + const OPAQUE: bool = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE; type Layout = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType as #pyo3_path::type_object::PyTypeInfo>::Layout; diff --git a/src/exceptions.rs b/src/exceptions.rs index 915fef6925d..03ffc30fcf3 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -125,6 +125,7 @@ macro_rules! import_exception_bound { $name, $name::type_object_raw, ::std::option::Option::Some(stringify!($module)), + false, $crate::impl_::pycell::PyStaticClassObject ); diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 496bcdb373a..1727da3e53d 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -2,19 +2,17 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::PyClassImpl; use crate::internal::get_slot::TP_ALLOC; -use crate::pycell::impl_::{ - PyClassObjectContents, PyClassObjectLayout, WrappedPyClassObjectContents, -}; +use crate::pycell::impl_::{PyClassObjectContents, PyObjectLayout}; use crate::types::PyType; use crate::{ffi, Borrowed, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; use std::marker::PhantomData; pub unsafe fn initialize_with_default(obj: *mut ffi::PyObject) { - let contents = T::Layout::contents_uninitialised(obj); - (*contents).write(WrappedPyClassObjectContents(PyClassObjectContents::new( - T::default(), - ))); + std::ptr::write( + PyObjectLayout::get_contents_ptr::(obj), + PyClassObjectContents::new(T::default()), + ); } /// Initializer for Python types. diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 37632873f04..55f9f47634f 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -320,7 +320,7 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. let _guard = TraverseGuard(class_object, PhantomData); - let instance = &*class_object.contents().0.value.get(); + let instance = &*class_object.get_ptr(); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 4fb34f2deee..5a2dfbf8b9b 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -3,7 +3,7 @@ use std::cell::UnsafeCell; use std::marker::PhantomData; -use std::mem::{ManuallyDrop, MaybeUninit}; +use std::mem::ManuallyDrop; use std::ptr::addr_of_mut; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -284,13 +284,6 @@ impl<'a, T: PyClassImpl> From<&'a mut PyClassObjectContents> ) )] pub trait PyClassObjectLayout: PyClassObjectBaseLayout { - /// Obtain a pointer to the contents of an uninitialized PyObject of this type. - /// - /// Safety: the provided object must have the layout that the implementation is expecting - unsafe fn contents_uninitialised( - obj: *mut ffi::PyObject, - ) -> *mut MaybeUninit>; - /// obtain a reference to the structure that contains the pyclass struct and associated metadata. fn contents(&self) -> &WrappedPyClassObjectContents; @@ -413,18 +406,6 @@ pub struct PyStaticClassObject { } impl PyClassObjectLayout for PyStaticClassObject { - unsafe fn contents_uninitialised( - obj: *mut ffi::PyObject, - ) -> *mut MaybeUninit> { - #[repr(C)] - struct PartiallyInitializedClassObject { - _ob_base: ::LayoutAsBase, - contents: MaybeUninit>, - } - let obj: *mut PartiallyInitializedClassObject = obj.cast(); - addr_of_mut!((*obj).contents) - } - fn contents(&self) -> &WrappedPyClassObjectContents { (&self.contents).into() } @@ -532,12 +513,6 @@ impl PyVariableClassObject { #[cfg(Py_3_12)] impl PyClassObjectLayout for PyVariableClassObject { - unsafe fn contents_uninitialised( - obj: *mut ffi::PyObject, - ) -> *mut MaybeUninit> { - Self::get_contents_of_obj(obj) as *mut MaybeUninit> - } - fn get_ptr(&self) -> *mut T { self.contents().0.value.get() } @@ -618,6 +593,65 @@ fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { value.try_into().expect("value should fit in Py_ssize_t") } +/// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). +#[doc(hidden)] +mod opaque_layout { + use super::PyClassObjectContents; + use crate::ffi; + use crate::impl_::pyclass::PyClassImpl; + + #[cfg(Py_3_12)] + pub fn get_contents_ptr( + obj: *mut ffi::PyObject, + ) -> *mut PyClassObjectContents { + #[cfg(Py_3_12)] + { + let type_obj = unsafe { ffi::Py_TYPE(obj) }; + assert!(!type_obj.is_null()); + let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; + assert!(!pointer.is_null()); + pointer as *mut PyClassObjectContents + } + + #[cfg(not(Py_3_12))] + panic!("opaque layout not supported until python 3.12"); + } +} + +/// Utilities for working with `PyObject` objects that utilise the standard layout for python extensions, +/// where the base class is placed at the beginning of a `repr(C)` struct. +#[doc(hidden)] +mod static_layout { + use crate::impl_::pyclass::{PyClassBaseType, PyClassImpl}; + + use super::PyClassObjectContents; + + #[repr(C)] + pub struct ClassObject { + pub ob_base: ::LayoutAsBase, + pub contents: PyClassObjectContents, + } +} + +pub(crate) struct PyObjectLayout {} + +impl PyObjectLayout { + /// Obtain a pointer to the contents of an PyObject of this type. + /// + /// Safety: the provided object must be valid and have the layout indicated by `T` + pub(crate) unsafe fn get_contents_ptr( + obj: *mut ffi::PyObject, + ) -> *mut PyClassObjectContents { + debug_assert!(!obj.is_null()); + if ::OPAQUE { + opaque_layout::get_contents_ptr(obj) + } else { + let obj: *mut static_layout::ClassObject = obj.cast(); + addr_of_mut!((*obj).contents) + } + } +} + #[cfg(test)] #[cfg(feature = "macros")] mod tests { diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index baa1df95b99..3187dac3e60 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,9 +1,9 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; -use crate::impl_::pyclass::{PyClassBaseType, PyClassImpl}; +use crate::impl_::pyclass::PyClassBaseType; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; -use crate::pycell::impl_::{PyClassObjectLayout, WrappedPyClassObjectContents}; +use crate::pycell::impl_::PyObjectLayout; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; use crate::{ffi::PyTypeObject, pycell::impl_::PyClassObjectContents}; @@ -166,10 +166,10 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; - let contents = ::Layout::contents_uninitialised(obj); - (*contents).write(WrappedPyClassObjectContents(PyClassObjectContents::new( - init, - ))); + std::ptr::write( + PyObjectLayout::get_contents_ptr::(obj), + PyClassObjectContents::new(init), + ); // Safety: obj is a valid pointer to an object of type `target_type`, which` is a known // subclass of `T` diff --git a/src/type_object.rs b/src/type_object.rs index 0aabcf82a1c..e7e366cb3f4 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -28,7 +28,7 @@ pub trait PySizedLayout: PyLayout + Sized {} /// /// This trait is marked unsafe because: /// - specifying the incorrect layout can lead to memory errors -/// - the return value of type_object must always point to the same PyTypeObject instance +/// - the return value of type_object must always point to the same `PyTypeObject` instance /// /// It is safely implemented by the `pyclass` macro. /// @@ -43,11 +43,16 @@ pub unsafe trait PyTypeInfo: Sized { /// Module name, if any. const MODULE: Option<&'static str>; + /// Whether classes that extend from this type must use the 'opaque type' extension mechanism + /// rather than using the standard mechanism of placing the data for this type at the beginning + /// of a new `repr(C)` struct + const OPAQUE: bool; + /// The type of object layout to use for ancestors or descendants of this type. /// should implement `PyClassObjectLayout` in order to actually use it as a layout. type Layout; - /// Returns the PyTypeObject instance for this type. + /// Returns the `PyTypeObject` instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; /// Returns the safe abstraction over the type object. diff --git a/src/types/any.rs b/src/types/any.rs index 8e689e8cc68..842e12d95e9 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -42,6 +42,7 @@ pyobject_native_type_info!( PyAny, pyobject_native_static_type_object!(ffi::PyBaseObject_Type), Some("builtins"), + false, PyStaticClassObject, #checkfunction=PyObject_Check ); diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index 6f0544db999..9ea2f2a4030 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -32,8 +32,8 @@ impl PyEllipsis { unsafe impl PyTypeInfo for PyEllipsis { const NAME: &'static str = "ellipsis"; - const MODULE: Option<&'static str> = None; + const OPAQUE: bool = false; type Layout = PyStaticClassObject; diff --git a/src/types/mod.rs b/src/types/mod.rs index 864568f4140..f35d7cd0cfa 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -153,10 +153,11 @@ macro_rules! pyobject_native_static_type_object( #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_info( - ($name:ty, $typeobject:expr, $module:expr, $layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $typeobject:expr, $module:expr, $opaque:expr, $layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { unsafe impl<$($generics,)*> $crate::type_object::PyTypeInfo for $name { const NAME: &'static str = stringify!($name); const MODULE: ::std::option::Option<&'static str> = $module; + const OPAQUE: bool = $opaque; type Layout = $layout; @@ -186,15 +187,15 @@ macro_rules! pyobject_native_type_info( #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_core { - ($name:ty, $typeobject:expr, #module=$module:expr, #layout=$layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $typeobject:expr, #module=$module:expr, #opaque=$opaque:expr, #layout=$layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { $crate::pyobject_native_type_named!($name $(;$generics)*); - $crate::pyobject_native_type_info!($name, $typeobject, $module, $layout $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_info!($name, $typeobject, $module, $opaque, $layout $(, #checkfunction=$checkfunction)? $(;$generics)*); }; ($name:ty, $typeobject:expr, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=$module, #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_core!($name, $typeobject, #module=$module, #opaque=false, #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); }; ($name:ty, $typeobject:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins"), #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins"), #opaque=false, #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); }; } diff --git a/src/types/none.rs b/src/types/none.rs index 67a3fc28e4d..86e80bbe2e9 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -31,8 +31,8 @@ impl PyNone { unsafe impl PyTypeInfo for PyNone { const NAME: &'static str = "NoneType"; - const MODULE: Option<&'static str> = None; + const OPAQUE: bool = false; type Layout = PyStaticClassObject; diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index 4fbb836d0b5..fe4e4556cb3 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -37,6 +37,7 @@ impl PyNotImplemented { unsafe impl PyTypeInfo for PyNotImplemented { const NAME: &'static str = "NotImplementedType"; const MODULE: Option<&'static str> = None; + const OPAQUE: bool = false; type Layout = PyStaticClassObject; diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 40c8169a08f..a0da756c411 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -21,6 +21,7 @@ pyobject_native_type_core!( PyType, pyobject_native_static_type_object!(ffi::PyType_Type), #module=::std::option::Option::Some("builtins"), + #opaque=true, #layout=crate::impl_::pycell::PyVariableClassObject, #checkfunction = ffi::PyType_Check ); From 0161b7aca76d1eb8b2c47b64ea62522d9576cc27 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 4 Nov 2024 21:42:09 +0000 Subject: [PATCH 22/41] migrate basicsize to PyObjectLayout --- src/pycell/impl_.rs | 39 ++++++++++++++----------------- src/pyclass/create_type_object.rs | 7 +++--- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 5a2dfbf8b9b..4a40111cc36 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -296,10 +296,6 @@ pub trait PyClassObjectLayout: PyClassObjectBaseLayout { /// obtain a reference to the data at the start of the PyObject. fn ob_base(&self) -> &::LayoutAsBase; - /// Used to set `PyType_Spec::basicsize` - /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) - fn basicsize() -> ffi::Py_ssize_t; - /// Gets the offset of the contents from the start of the struct in bytes. fn contents_offset() -> PyObjectOffset; @@ -422,14 +418,6 @@ impl PyClassObjectLayout for PyStaticClassObject { &self.ob_base } - fn basicsize() -> ffi::Py_ssize_t { - let size = std::mem::size_of::(); - - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - size.try_into().expect("size should fit in Py_ssize_t") - } - fn contents_offset() -> PyObjectOffset { PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( PyStaticClassObject, @@ -531,12 +519,6 @@ impl PyClassObjectLayout for PyVariableClassObject { .expect("should be able to cast PyClassObjectContents pointer") } - fn basicsize() -> ffi::Py_ssize_t { - let size = std::mem::size_of::>(); - // negative to indicate 'extra' space that cpython will allocate for us - -usize_to_py_ssize(size) - } - fn contents_offset() -> PyObjectOffset { PyObjectOffset::Relative(0) } @@ -626,6 +608,7 @@ mod static_layout { use super::PyClassObjectContents; + // The layout of a `PyObject` that uses the static layout #[repr(C)] pub struct ClassObject { pub ob_base: ::LayoutAsBase, @@ -636,7 +619,7 @@ mod static_layout { pub(crate) struct PyObjectLayout {} impl PyObjectLayout { - /// Obtain a pointer to the contents of an PyObject of this type. + /// Obtain a pointer to the contents of an `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` pub(crate) unsafe fn get_contents_ptr( @@ -650,6 +633,18 @@ impl PyObjectLayout { addr_of_mut!((*obj).contents) } } + + /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` + /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) + pub(crate) fn basicsize() -> ffi::Py_ssize_t { + if ::OPAQUE { + // negative to indicate 'extra' space that python will allocate + // specifically for `T` excluding the base class + -usize_to_py_ssize(std::mem::size_of::>()) + } else { + usize_to_py_ssize(std::mem::size_of::>()) + } + } } #[cfg(test)] @@ -713,13 +708,13 @@ mod tests { #[test] fn test_inherited_size() { - let base_size = PyStaticClassObject::::basicsize(); + let base_size = PyObjectLayout::basicsize::(); assert!(base_size > 0); // negative indicates variable sized assert_eq!( base_size, - PyStaticClassObject::::basicsize() + PyObjectLayout::basicsize::() ); - assert!(base_size < PyStaticClassObject::::basicsize()); + assert!(base_size < PyObjectLayout::basicsize::()); } fn assert_mutable>() {} diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index fe0bdbfe5ce..5c2169ceb07 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -4,14 +4,13 @@ use crate::{ impl_::{ pyclass::{ assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, - tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassImpl, PyClassItemsIter, - PyObjectOffset, + tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassItemsIter, PyObjectOffset, }, pymethods::{Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter, _call_clear}, trampoline::trampoline, }, internal_tricks::ptr_from_ref, - pycell::impl_::PyClassObjectLayout, + pycell::impl_::PyObjectLayout, types::{typeobject::PyTypeMethods, PyType}, Py, PyClass, PyResult, PyTypeInfo, Python, }; @@ -95,7 +94,7 @@ where T::items_iter(), T::NAME, T::MODULE, - ::Layout::basicsize(), + PyObjectLayout::basicsize::(), ) } } From 7b69fa3220820b6126d1bfda6a7abc4ffed26560 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 4 Nov 2024 23:27:59 +0000 Subject: [PATCH 23/41] move base types into submodules --- src/impl_/pycell.rs | 4 +- src/instance.rs | 5 +- src/pycell.rs | 11 ++-- src/pycell/impl_.rs | 135 ++++++++++++++++++++++++++------------------ 4 files changed, 91 insertions(+), 64 deletions(-) diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index 850ab1492c5..2d1ae7268da 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,5 +1,5 @@ //! Externally-accessible implementation of pycell pub use crate::pycell::impl_::{ - GetBorrowChecker, PyClassMutability, PyClassObjectBase, PyClassObjectBaseLayout, - PyStaticClassObject, PyVariableClassObject, PyVariableClassObjectBase, + opaque_layout::PyVariableClassObjectBase, static_layout::PyClassObjectBase, GetBorrowChecker, + PyClassMutability, PyClassObjectBaseLayout, PyStaticClassObject, PyVariableClassObject, }; diff --git a/src/instance.rs b/src/instance.rs index 093b8b17fcf..73e76fa017c 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2,7 +2,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::impl_::pyclass::PyClassImpl; use crate::internal_tricks::ptr_from_ref; -use crate::pycell::impl_::PyClassObjectLayout; +use crate::pycell::impl_::{PyObjectHandle, PyObjectLayout}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; @@ -1290,8 +1290,9 @@ where where T: PyClass + Sync, { + let obj = self.as_ptr(); // Safety: The class itself is frozen and `Sync` - unsafe { &*self.get_class_object().get_ptr() } + unsafe { &*PyObjectLayout::get_data_ptr::(obj) } } /// Get a view on the underlying `PyClass` contents. diff --git a/src/pycell.rs b/src/pycell.rs index 057b4c2b77f..68e834a6596 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -208,7 +208,7 @@ use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::{PyClassBorrowChecker, PyClassObjectBaseLayout, PyClassObjectLayout}; +use impl_::{PyClassBorrowChecker, PyClassObjectBaseLayout, PyClassObjectLayout, PyObjectLayout}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// @@ -436,7 +436,8 @@ impl Deref for PyRef<'_, T> { #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_class_object().get_ptr() } + let obj = self.inner.as_ptr(); + unsafe { &*PyObjectLayout::get_data_ptr::(obj) } } } @@ -620,14 +621,16 @@ impl> Deref for PyRefMut<'_, T> { #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_class_object().get_ptr() } + let obj = self.inner.as_ptr(); + unsafe { &*PyObjectLayout::get_data_ptr::(obj) } } } impl> DerefMut for PyRefMut<'_, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.inner.get_class_object().get_ptr() } + let obj = self.inner.as_ptr(); + unsafe { &mut *PyObjectLayout::get_data_ptr::(obj) } } } diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 4a40111cc36..f9ad88afd16 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -208,36 +208,6 @@ where } } -/// Base layout of PyClassObject with a known sized base type. -/// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. -#[doc(hidden)] -#[repr(C)] -pub struct PyClassObjectBase { - ob_base: T, -} - -unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} - -/// Base layout of PyClassObject with an unknown sized base type. -/// Corresponds to [PyVarObject](https://docs.python.org/3/c-api/structures.html#c.PyVarObject) from the C API. -#[doc(hidden)] -#[repr(C)] -pub struct PyVariableClassObjectBase { - ob_base: ffi::PyVarObject, -} - -unsafe impl PyLayout for PyVariableClassObjectBase {} - -impl PyClassObjectBaseLayout for PyVariableClassObjectBase { - fn ensure_threadsafe(&self) {} - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - Ok(()) - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - tp_dealloc(py, slf, T::type_object_raw(py)); - } -} - /// functionality common to all PyObjects regardless of the layout #[doc(hidden)] pub trait PyClassObjectBaseLayout: PyLayout { @@ -308,21 +278,7 @@ pub trait PyClassObjectLayout: PyClassObjectBaseLayout { fn borrow_checker(&self) -> &::Checker; } -impl PyClassObjectBaseLayout for PyClassObjectBase -where - U: PySizedLayout, - T: PyTypeInfo, -{ - fn ensure_threadsafe(&self) {} - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - Ok(()) - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - tp_dealloc(py, slf, T::type_object_raw(py)); - } -} - -/// Implementation of tp_dealloc. +/// Implementation of `tp_dealloc`. /// # Safety /// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. /// - obj must not be used after this call (as it will be freed). @@ -577,10 +533,32 @@ fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { /// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). #[doc(hidden)] -mod opaque_layout { - use super::PyClassObjectContents; - use crate::ffi; +pub(crate) mod opaque_layout { + use super::{tp_dealloc, PyClassObjectBaseLayout, PyClassObjectContents}; use crate::impl_::pyclass::PyClassImpl; + use crate::pycell::PyBorrowError; + use crate::type_object::PyLayout; + use crate::{ffi, PyTypeInfo, Python}; + + /// Base layout of `PyClassObject` with an unknown sized base type. + /// Corresponds to [PyVarObject](https://docs.python.org/3/c-api/structures.html#c.PyVarObject) from the C API. + #[doc(hidden)] + #[repr(C)] + pub struct PyVariableClassObjectBase { + ob_base: ffi::PyVarObject, + } + + unsafe impl PyLayout for PyVariableClassObjectBase {} + + impl PyClassObjectBaseLayout for PyVariableClassObjectBase { + fn ensure_threadsafe(&self) {} + fn check_threadsafe(&self) -> Result<(), PyBorrowError> { + Ok(()) + } + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + tp_dealloc(py, slf, T::type_object_raw(py)); + } + } #[cfg(Py_3_12)] pub fn get_contents_ptr( @@ -603,10 +581,16 @@ mod opaque_layout { /// Utilities for working with `PyObject` objects that utilise the standard layout for python extensions, /// where the base class is placed at the beginning of a `repr(C)` struct. #[doc(hidden)] -mod static_layout { - use crate::impl_::pyclass::{PyClassBaseType, PyClassImpl}; +pub(crate) mod static_layout { + use crate::{ + ffi, + impl_::pyclass::{PyClassBaseType, PyClassImpl}, + pycell::PyBorrowError, + type_object::{PyLayout, PySizedLayout}, + PyTypeInfo, Python, + }; - use super::PyClassObjectContents; + use super::{tp_dealloc, PyClassObjectBaseLayout, PyClassObjectContents}; // The layout of a `PyObject` that uses the static layout #[repr(C)] @@ -614,12 +598,36 @@ mod static_layout { pub ob_base: ::LayoutAsBase, pub contents: PyClassObjectContents, } + + /// Base layout of PyClassObject with a known sized base type. + /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. + #[doc(hidden)] + #[repr(C)] + pub struct PyClassObjectBase { + ob_base: T, + } + + unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} + + impl PyClassObjectBaseLayout for PyClassObjectBase + where + U: PySizedLayout, + T: PyTypeInfo, + { + fn ensure_threadsafe(&self) {} + fn check_threadsafe(&self) -> Result<(), PyBorrowError> { + Ok(()) + } + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + tp_dealloc(py, slf, T::type_object_raw(py)); + } + } } pub(crate) struct PyObjectLayout {} impl PyObjectLayout { - /// Obtain a pointer to the contents of an `PyObject` of type `T`. + /// Obtain a pointer to the contents of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` pub(crate) unsafe fn get_contents_ptr( @@ -634,6 +642,24 @@ impl PyObjectLayout { } } + /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. + /// + /// Safety: the provided object must be valid and have the layout indicated by `T` + pub(crate) unsafe fn get_data_ptr(obj: *mut ffi::PyObject) -> *mut T { + let contents = PyObjectLayout::get_contents_ptr::(obj); + (*contents).value.get() + } + + /// obtain a reference to the data at the start of the `PyObject`. + /// + /// Safety: the provided object must be valid and have the layout indicated by `T` + pub(crate) unsafe fn ob_base( + obj: *mut ffi::PyObject, + ) -> *mut ::LayoutAsBase { + // the base layout is always at the beginning of the `PyObject` so the pointer can simply be casted + obj as *mut ::LayoutAsBase + } + /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) pub(crate) fn basicsize() -> ffi::Py_ssize_t { @@ -710,10 +736,7 @@ mod tests { fn test_inherited_size() { let base_size = PyObjectLayout::basicsize::(); assert!(base_size > 0); // negative indicates variable sized - assert_eq!( - base_size, - PyObjectLayout::basicsize::() - ); + assert_eq!(base_size, PyObjectLayout::basicsize::()); assert!(base_size < PyObjectLayout::basicsize::()); } From 29fc1af0bac2fe05908700ca1766740802b57a48 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 4 Nov 2024 23:53:49 +0000 Subject: [PATCH 24/41] migrate more functionality --- src/impl_/pyclass.rs | 19 +++-- src/instance.rs | 2 +- src/pycell/impl_.rs | 178 ++++++++++++++++++++++++++----------------- 3 files changed, 117 insertions(+), 82 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index b96be514568..5e1a6492142 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1,5 +1,3 @@ -#[cfg(Py_3_12)] -use crate::pycell::impl_::PyClassObjectContents; use crate::{ conversion::IntoPyObject, exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError}, @@ -10,7 +8,10 @@ use crate::{ pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, - pycell::{impl_::PyClassObjectLayout, PyBorrowError}, + pycell::{ + impl_::{PyClassObjectLayout, PyObjectLayout}, + PyBorrowError, + }, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, }; @@ -31,13 +32,13 @@ pub use lazy_type_object::LazyTypeObject; /// Gets the offset of the dictionary from the start of the object in bytes. #[inline] pub fn dict_offset() -> PyObjectOffset { - ::Layout::dict_offset() + PyObjectLayout::dict_offset::() } /// Gets the offset of the weakref list from the start of the object in bytes. #[inline] pub fn weaklist_offset() -> PyObjectOffset { - ::Layout::weaklist_offset() + PyObjectLayout::weaklist_offset::() } /// Represents the `__dict__` field for `#[pyclass]`. @@ -1229,7 +1230,7 @@ pub unsafe trait OffsetCalculator { // Used in generated implementations of OffsetCalculator pub fn subclass_offset() -> PyObjectOffset { - ::Layout::contents_offset() + PyObjectLayout::contents_offset::() } // Used in generated implementations of OffsetCalculator @@ -1546,10 +1547,8 @@ where PyObjectOffset::Absolute(offset) => (obj.cast::(), offset), #[cfg(Py_3_12)] PyObjectOffset::Relative(offset) => { - let class_ptr = obj.cast::<::Layout>(); - // Safety: the object `obj` must have the layout `ClassT::Layout` - let class_obj = unsafe { &mut *class_ptr }; - let contents = (&mut class_obj.contents_mut().0) as *mut PyClassObjectContents; + // Safety: obj must be a valid `PyObject` of type `ClassT` + let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; (contents.cast::(), offset) } #[cfg(not(Py_3_12))] diff --git a/src/instance.rs b/src/instance.rs index 73e76fa017c..9240b389953 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2,7 +2,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::impl_::pyclass::PyClassImpl; use crate::internal_tricks::ptr_from_ref; -use crate::pycell::impl_::{PyObjectHandle, PyObjectLayout}; +use crate::pycell::impl_::PyObjectLayout; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index f9ad88afd16..5e7594c177f 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -257,24 +257,12 @@ pub trait PyClassObjectLayout: PyClassObjectBaseLayout { /// obtain a reference to the structure that contains the pyclass struct and associated metadata. fn contents(&self) -> &WrappedPyClassObjectContents; - /// obtain a mutable reference to the structure that contains the pyclass struct and associated metadata. - fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents; - /// obtain a pointer to the pyclass struct. fn get_ptr(&self) -> *mut T; /// obtain a reference to the data at the start of the PyObject. fn ob_base(&self) -> &::LayoutAsBase; - /// Gets the offset of the contents from the start of the struct in bytes. - fn contents_offset() -> PyObjectOffset; - - /// Gets the offset of the dictionary from the start of the struct in bytes. - fn dict_offset() -> PyObjectOffset; - - /// Gets the offset of the weakref list from the start of the struct in bytes. - fn weaklist_offset() -> PyObjectOffset; - fn borrow_checker(&self) -> &::Checker; } @@ -362,10 +350,6 @@ impl PyClassObjectLayout for PyStaticClassObject { (&self.contents).into() } - fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents { - (&mut self.contents).into() - } - fn get_ptr(&self) -> *mut T { self.contents.value.get() } @@ -374,31 +358,6 @@ impl PyClassObjectLayout for PyStaticClassObject { &self.ob_base } - fn contents_offset() -> PyObjectOffset { - PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( - PyStaticClassObject, - contents - ))) - } - - fn dict_offset() -> PyObjectOffset { - use memoffset::offset_of; - - let offset = offset_of!(PyStaticClassObject, contents) - + offset_of!(PyClassObjectContents, dict); - - PyObjectOffset::Absolute(usize_to_py_ssize(offset)) - } - - fn weaklist_offset() -> PyObjectOffset { - use memoffset::offset_of; - - let offset = offset_of!(PyStaticClassObject, contents) - + offset_of!(PyClassObjectContents, weakref); - - PyObjectOffset::Absolute(usize_to_py_ssize(offset)) - } - fn borrow_checker(&self) -> &::Checker { // Safety: T::Layout must be PyStaticClassObject let slf: &T::Layout = unsafe { std::mem::transmute(self) }; @@ -425,8 +384,8 @@ where } unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. - let class_object = &mut *(slf.cast::()); - class_object.contents_mut().0.dealloc(py, slf); + let contents = unsafe { PyObjectLayout::get_contents_ptr::(slf) }; + (*contents).dealloc(py, slf); ::LayoutAsBase::tp_dealloc(py, slf) } } @@ -470,29 +429,6 @@ impl PyClassObjectLayout for PyVariableClassObject { .expect("should be able to cast PyClassObjectContents pointer") } - fn contents_mut(&mut self) -> &mut WrappedPyClassObjectContents { - unsafe { self.get_contents_ptr().as_mut() } - .expect("should be able to cast PyClassObjectContents pointer") - } - - fn contents_offset() -> PyObjectOffset { - PyObjectOffset::Relative(0) - } - - fn dict_offset() -> PyObjectOffset { - PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( - PyClassObjectContents, - dict - ))) - } - - fn weaklist_offset() -> PyObjectOffset { - PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( - PyClassObjectContents, - weakref - ))) - } - fn borrow_checker(&self) -> &::Checker { // Safety: T::Layout must be PyStaticClassObject let slf: &T::Layout = unsafe { std::mem::transmute(self) }; @@ -519,8 +455,8 @@ where } unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. - let class_object = &mut *(slf.cast::()); - class_object.contents_mut().0.dealloc(py, slf); + let contents = unsafe { PyObjectLayout::get_contents_ptr::(slf) }; + (*contents).dealloc(py, slf); ::LayoutAsBase::tp_dealloc(py, slf) } } @@ -574,6 +510,12 @@ pub(crate) mod opaque_layout { } #[cfg(not(Py_3_12))] + panic_unsupported(); + } + + #[inline(always)] + #[cfg(not(Py_3_12))] + fn panic_unsupported() { panic!("opaque layout not supported until python 3.12"); } } @@ -664,13 +606,107 @@ impl PyObjectLayout { /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) pub(crate) fn basicsize() -> ffi::Py_ssize_t { if ::OPAQUE { - // negative to indicate 'extra' space that python will allocate - // specifically for `T` excluding the base class - -usize_to_py_ssize(std::mem::size_of::>()) + #[cfg(Py_3_12)] + { + // negative to indicate 'extra' space that python will allocate + // specifically for `T` excluding the base class + -usize_to_py_ssize(std::mem::size_of::>()) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); } else { usize_to_py_ssize(std::mem::size_of::>()) } } + + /// Gets the offset of the contents from the start of the struct in bytes. + pub(crate) fn contents_offset() -> PyObjectOffset { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(0) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( + static_layout::ClassObject, + contents + ))) + } + } + + /// Gets the offset of the dictionary from the start of the struct in bytes. + pub(crate) fn dict_offset() -> PyObjectOffset { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + dict + ))) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + let offset = memoffset::offset_of!(static_layout::ClassObject, contents) + + memoffset::offset_of!(PyClassObjectContents, dict); + + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) + } + } + + /// Gets the offset of the weakref list from the start of the struct in bytes. + pub(crate) fn weaklist_offset() -> PyObjectOffset { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + weakref + ))) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + let offset = memoffset::offset_of!(static_layout::ClassObject, contents) + + memoffset::offset_of!(PyClassObjectContents, weakref); + + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) + } + } +} + +/// A wrapper around PyObject to provide [PyObjectLayout] functionality. +pub(crate) struct PyObjectHandle<'a, T: PyClassImpl>(*mut ffi::PyObject, PhantomData<&'a T>); + +impl<'a, T: PyClassImpl> PyObjectHandle<'a, T> { + /// Safety: obj must point to a valid PyObject with the type `T` + pub unsafe fn new(obj: *mut ffi::PyObject) -> Self { + debug_assert!(!obj.is_null()); + PyObjectHandle(obj, PhantomData) + } + + pub fn contents(&'a self) -> &'a PyClassObjectContents { + unsafe { &*PyObjectLayout::get_contents_ptr::(self.0) } + } + + pub fn data(&'a self) -> &'a T { + unsafe { &*PyObjectLayout::get_data_ptr::(self.0) } + } + + pub fn ensure_threadsafe(&self) { + let contents = unsafe { &(*PyObjectLayout::get_contents_ptr::(self.0)) }; + contents.thread_checker.ensure(); + let base = unsafe { &(*PyObjectLayout::ob_base::(self.0)) }; + base.ensure_threadsafe(); + } + + pub fn borrow_checker(&self) {} } #[cfg(test)] From aada7d8e1f7900e2ec145b959e363c7cbae92f29 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 8 Nov 2024 20:43:14 +0000 Subject: [PATCH 25/41] migrate borrow checker to PyObjectLayout --- src/impl_/pyclass.rs | 13 ++++--- src/impl_/pymethods.rs | 16 ++++---- src/instance.rs | 14 +++++++ src/pycell.rs | 24 ++++++------ src/pycell/impl_.rs | 85 +++++++++++++++++------------------------- 5 files changed, 77 insertions(+), 75 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 5e1a6492142..b78c1c6895d 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -158,22 +158,22 @@ unsafe impl Sync for PyClassItems {} /// Users are discouraged from implementing this trait manually; it is a PyO3 implementation detail /// and may be changed at any time. pub trait PyClassImpl: Sized + 'static { - /// #[pyclass(subclass)] + /// `#[pyclass(subclass)]` const IS_BASETYPE: bool = false; - /// #[pyclass(extends=...)] + /// `#[pyclass(extends=...)]` const IS_SUBCLASS: bool = false; - /// #[pyclass(mapping)] + /// `#[pyclass(mapping)]` const IS_MAPPING: bool = false; - /// #[pyclass(sequence)] + /// `#[pyclass(sequence)]` const IS_SEQUENCE: bool = false; /// Description of how this class is laid out in memory type Layout: PyClassObjectLayout; - /// Base class + /// Base class (the direct parent configured via `#[pyclass(extends=...)]`) type BaseType: PyTypeInfo + PyClassBaseType; /// Immutable or mutable @@ -1119,7 +1119,8 @@ impl PyClassThreadChecker for ThreadCheckerImpl { private_impl! {} } -/// Trait denoting that this class is suitable to be used as a base type for PyClass. +/// Trait denoting that this class is suitable to be used as a base type for PyClass +/// (meaning it can be used with `#[pyclass(extends=...)]`). #[cfg_attr( all(diagnostic_namespace, Py_LIMITED_API), diagnostic::on_unimplemented( diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 55f9f47634f..ecb749cc38a 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -4,7 +4,7 @@ use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; use crate::impl_::pycell::PyClassObjectBaseLayout; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; -use crate::pycell::impl_::{PyClassBorrowChecker as _, PyClassObjectLayout}; +use crate::pycell::impl_::{PyClassBorrowChecker as _, PyObjectLayout}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; @@ -300,6 +300,7 @@ where return super_retval; } + let raw_obj = &*slf; // SAFETY: `slf` is a valid Python object pointer to a class object of type T, and // traversal is running so no mutations can occur. let class_object: &::Layout = &*slf.cast(); @@ -309,18 +310,19 @@ where // do not traverse them if not on their owning thread :( if class_object.check_threadsafe().is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread - && class_object.borrow_checker().try_borrow().is_ok() { - struct TraverseGuard<'a, U: PyClassImpl, V: PyClassObjectLayout>(&'a V, PhantomData); - impl> Drop for TraverseGuard<'_, U, V> { + && PyObjectLayout::get_borrow_checker::(raw_obj).try_borrow().is_ok() { + struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); + impl Drop for TraverseGuard<'_, Cls> { fn drop(&mut self) { - self.0.borrow_checker().release_borrow() + let borrow_checker = unsafe {PyObjectLayout::get_borrow_checker::(self.0)}; + borrow_checker.release_borrow(); } } // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. - let _guard = TraverseGuard(class_object, PhantomData); - let instance = &*class_object.get_ptr(); + let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); + let instance = unsafe {PyObjectLayout::get_data::(raw_obj)}; let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index 9240b389953..890fdd0dd84 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -466,6 +466,11 @@ where pub(crate) fn get_class_object(&self) -> &::Layout { self.1.get_class_object() } + + #[inline] + pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { + self.1.get_raw_object() + } } impl std::fmt::Debug for Bound<'_, T> { @@ -1303,6 +1308,15 @@ where // ::Layout object unsafe { &*class_object } } + + /// Get a reference to the underlying `PyObject` + #[inline] + pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { + let obj = self.as_ptr(); + assert!(!obj.is_null()); + // Safety: obj is a valid pointer to a PyObject + unsafe { &*obj } + } } impl Py { diff --git a/src/pycell.rs b/src/pycell.rs index 68e834a6596..095ad6d2d4d 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -208,7 +208,7 @@ use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::{PyClassBorrowChecker, PyClassObjectBaseLayout, PyClassObjectLayout, PyObjectLayout}; +use impl_::{PyClassBorrowChecker, PyClassObjectBaseLayout, PyObjectLayout}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// @@ -310,7 +310,9 @@ impl<'py, T: PyClass> PyRef<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let cell = obj.get_class_object(); cell.ensure_threadsafe(); - cell.borrow_checker() + let raw_obj = obj.get_raw_object(); + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; + borrow_checker .try_borrow() .map(|_| Self { inner: obj.clone() }) } @@ -443,10 +445,9 @@ impl Deref for PyRef<'_, T> { impl Drop for PyRef<'_, T> { fn drop(&mut self) { - self.inner - .get_class_object() - .borrow_checker() - .release_borrow() + let obj = self.inner.get_raw_object(); + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(obj) }; + borrow_checker.release_borrow(); } } @@ -567,7 +568,9 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let cell = obj.get_class_object(); cell.ensure_threadsafe(); - cell.borrow_checker() + let raw_obj = obj.get_raw_object(); + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; + borrow_checker .try_borrow_mut() .map(|_| Self { inner: obj.clone() }) } @@ -636,10 +639,9 @@ impl> DerefMut for PyRefMut<'_, T> { impl> Drop for PyRefMut<'_, T> { fn drop(&mut self) { - self.inner - .get_class_object() - .borrow_checker() - .release_borrow_mut() + let obj = self.inner.get_raw_object(); + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(obj) }; + borrow_checker.release_borrow_mut(); } } diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 5e7594c177f..d0402cee83d 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -104,15 +104,13 @@ pub struct BorrowChecker(BorrowFlag); pub trait PyClassBorrowChecker { /// Initial value for self fn new() -> Self; - /// Increments immutable borrow count, if possible fn try_borrow(&self) -> Result<(), PyBorrowError>; - /// Decrements immutable borrow count fn release_borrow(&self); /// Increments mutable borrow count, if possible fn try_borrow_mut(&self) -> Result<(), PyBorrowMutError>; - /// Decremements mutable borrow count + /// Decrements mutable borrow count fn release_borrow_mut(&self); } @@ -180,31 +178,36 @@ impl PyClassBorrowChecker for BorrowChecker { } pub trait GetBorrowChecker { - fn borrow_checker( - class_object: &T::Layout, - ) -> &::Checker; + fn borrow_checker(obj: &ffi::PyObject) + -> &::Checker; } impl> GetBorrowChecker for MutableClass { - fn borrow_checker(class_object: &T::Layout) -> &BorrowChecker { - &class_object.contents().0.borrow_checker + fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { + let obj = (obj as *const ffi::PyObject).cast_mut(); + let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; + unsafe { &(*contents).borrow_checker } } } impl> GetBorrowChecker for ImmutableClass { - fn borrow_checker(class_object: &T::Layout) -> &EmptySlot { - &class_object.contents().0.borrow_checker + fn borrow_checker(obj: &ffi::PyObject) -> &EmptySlot { + let obj = (obj as *const ffi::PyObject).cast_mut(); + let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; + unsafe { &(*contents).borrow_checker } } } -impl, M: PyClassMutability> GetBorrowChecker - for ExtendsMutableAncestor +impl GetBorrowChecker for ExtendsMutableAncestor where - T::BaseType: PyClassImpl + PyClassBaseType::Layout>, + T: PyClassImpl, + M: PyClassMutability, + T::BaseType: PyClassImpl, ::PyClassMutability: PyClassMutability, { - fn borrow_checker(class_object: &T::Layout) -> &BorrowChecker { - <::PyClassMutability as GetBorrowChecker>::borrow_checker(class_object.ob_base()) + fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { + // the same PyObject pointer can be re-interpreted as the base/parent type + <::PyClassMutability as GetBorrowChecker>::borrow_checker(obj) } } @@ -256,14 +259,6 @@ impl<'a, T: PyClassImpl> From<&'a mut PyClassObjectContents> pub trait PyClassObjectLayout: PyClassObjectBaseLayout { /// obtain a reference to the structure that contains the pyclass struct and associated metadata. fn contents(&self) -> &WrappedPyClassObjectContents; - - /// obtain a pointer to the pyclass struct. - fn get_ptr(&self) -> *mut T; - - /// obtain a reference to the data at the start of the PyObject. - fn ob_base(&self) -> &::LayoutAsBase; - - fn borrow_checker(&self) -> &::Checker; } /// Implementation of `tp_dealloc`. @@ -275,6 +270,7 @@ unsafe fn tp_dealloc(py: Python<'_>, obj: *mut ffi::PyObject, type_ptr: *mut ffi // at runtime? To be investigated. let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); + // TODO(matt): is this correct? // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); @@ -349,20 +345,6 @@ impl PyClassObjectLayout for PyStaticClassObject { fn contents(&self) -> &WrappedPyClassObjectContents { (&self.contents).into() } - - fn get_ptr(&self) -> *mut T { - self.contents.value.get() - } - - fn ob_base(&self) -> &::LayoutAsBase { - &self.ob_base - } - - fn borrow_checker(&self) -> &::Checker { - // Safety: T::Layout must be PyStaticClassObject - let slf: &T::Layout = unsafe { std::mem::transmute(self) }; - T::PyClassMutability::borrow_checker(slf) - } } unsafe impl PyLayout for PyStaticClassObject {} @@ -416,24 +398,10 @@ impl PyVariableClassObject { #[cfg(Py_3_12)] impl PyClassObjectLayout for PyVariableClassObject { - fn get_ptr(&self) -> *mut T { - self.contents().0.value.get() - } - - fn ob_base(&self) -> &::LayoutAsBase { - &self.ob_base - } - fn contents(&self) -> &WrappedPyClassObjectContents { unsafe { self.get_contents_ptr().cast_const().as_ref() } .expect("should be able to cast PyClassObjectContents pointer") } - - fn borrow_checker(&self) -> &::Checker { - // Safety: T::Layout must be PyStaticClassObject - let slf: &T::Layout = unsafe { std::mem::transmute(self) }; - T::PyClassMutability::borrow_checker(slf) - } } unsafe impl PyLayout for PyVariableClassObject {} @@ -566,6 +534,7 @@ pub(crate) mod static_layout { } } +/// Functions for working with `PyObject`s pub(crate) struct PyObjectLayout {} impl PyObjectLayout { @@ -592,6 +561,18 @@ impl PyObjectLayout { (*contents).value.get() } + pub(crate) unsafe fn get_data(obj: &ffi::PyObject) -> &T { + let obj_ref = (obj as *const ffi::PyObject).cast_mut(); + let data_ptr = unsafe { PyObjectLayout::get_data_ptr::(obj_ref) }; + unsafe { &*data_ptr } + } + + pub(crate) unsafe fn get_borrow_checker( + obj: &ffi::PyObject, + ) -> &::Checker { + T::PyClassMutability::borrow_checker(obj) + } + /// obtain a reference to the data at the start of the `PyObject`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` @@ -682,8 +663,10 @@ impl PyObjectLayout { } /// A wrapper around PyObject to provide [PyObjectLayout] functionality. +#[allow(unused)] pub(crate) struct PyObjectHandle<'a, T: PyClassImpl>(*mut ffi::PyObject, PhantomData<&'a T>); +#[allow(unused)] impl<'a, T: PyClassImpl> PyObjectHandle<'a, T> { /// Safety: obj must point to a valid PyObject with the type `T` pub unsafe fn new(obj: *mut ffi::PyObject) -> Self { From 39cb11297ff8dbfc6e094b8bbcb28b7e0999c4db Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 8 Nov 2024 20:57:07 +0000 Subject: [PATCH 26/41] remove old method --- src/pycell/impl_.rs | 73 +++++++-------------------------------------- 1 file changed, 10 insertions(+), 63 deletions(-) diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index d0402cee83d..21a06018caf 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -223,43 +223,9 @@ pub trait PyClassObjectBaseLayout: PyLayout { unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); } -/// Allow [PyClassObjectLayout] to have public visibility without leaking the structure of [PyClassObjectContents]. -#[doc(hidden)] -#[repr(transparent)] -pub struct WrappedPyClassObjectContents(pub(crate) PyClassObjectContents); - -impl<'a, T: PyClassImpl> From<&'a PyClassObjectContents> - for &'a WrappedPyClassObjectContents -{ - fn from(value: &'a PyClassObjectContents) -> &'a WrappedPyClassObjectContents { - // Safety: Wrapped struct must use repr(transparent) - unsafe { std::mem::transmute(value) } - } -} - -impl<'a, T: PyClassImpl> From<&'a mut PyClassObjectContents> - for &'a mut WrappedPyClassObjectContents -{ - fn from(value: &'a mut PyClassObjectContents) -> &'a mut WrappedPyClassObjectContents { - // Safety: Wrapped struct must use repr(transparent) - unsafe { std::mem::transmute(value) } - } -} - /// Functionality required for creating and managing the memory associated with a pyclass annotated struct. #[doc(hidden)] -#[cfg_attr( - all(diagnostic_namespace), - diagnostic::on_unimplemented( - message = "the class layout is not valid", - label = "required for `#[pyclass(extends=...)]`", - note = "the python version being built against influences which layouts are valid", - ) -)] -pub trait PyClassObjectLayout: PyClassObjectBaseLayout { - /// obtain a reference to the structure that contains the pyclass struct and associated metadata. - fn contents(&self) -> &WrappedPyClassObjectContents; -} +pub trait PyClassObjectLayout: PyClassObjectBaseLayout {} /// Implementation of `tp_dealloc`. /// # Safety @@ -341,11 +307,7 @@ pub struct PyStaticClassObject { contents: PyClassObjectContents, } -impl PyClassObjectLayout for PyStaticClassObject { - fn contents(&self) -> &WrappedPyClassObjectContents { - (&self.contents).into() - } -} +impl PyClassObjectLayout for PyStaticClassObject {} unsafe impl PyLayout for PyStaticClassObject {} impl PySizedLayout for PyStaticClassObject {} @@ -381,28 +343,8 @@ pub struct PyVariableClassObject { ob_base: ::LayoutAsBase, } -impl PyVariableClassObject { - #[cfg(Py_3_12)] - fn get_contents_of_obj(obj: *mut ffi::PyObject) -> *mut WrappedPyClassObjectContents { - // https://peps.python.org/pep-0697/ - let type_obj = unsafe { ffi::Py_TYPE(obj) }; - let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; - pointer as *mut WrappedPyClassObjectContents - } - - #[cfg(Py_3_12)] - fn get_contents_ptr(&self) -> *mut WrappedPyClassObjectContents { - Self::get_contents_of_obj(self as *const PyVariableClassObject as *mut ffi::PyObject) - } -} - #[cfg(Py_3_12)] -impl PyClassObjectLayout for PyVariableClassObject { - fn contents(&self) -> &WrappedPyClassObjectContents { - unsafe { self.get_contents_ptr().cast_const().as_ref() } - .expect("should be able to cast PyClassObjectContents pointer") - } -} +impl PyClassObjectLayout for PyVariableClassObject {} unsafe impl PyLayout for PyVariableClassObject {} @@ -412,11 +354,15 @@ where ::LayoutAsBase: PyClassObjectBaseLayout, { fn ensure_threadsafe(&self) { - self.contents().0.thread_checker.ensure(); + let obj_ptr = self as *const Self as *mut ffi::PyObject; + let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj_ptr) }; + contents.thread_checker.ensure(); self.ob_base.ensure_threadsafe(); } fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - if !self.contents().0.thread_checker.check() { + let obj_ptr = self as *const Self as *mut ffi::PyObject; + let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj_ptr) }; + if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } self.ob_base.check_threadsafe() @@ -470,6 +416,7 @@ pub(crate) mod opaque_layout { ) -> *mut PyClassObjectContents { #[cfg(Py_3_12)] { + // TODO(matt): this needs to be ::type_object_raw(py) let type_obj = unsafe { ffi::Py_TYPE(obj) }; assert!(!type_obj.is_null()); let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; From e7eeab99970e3165c660e54a2c08fca23cf648fb Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 9 Nov 2024 00:34:27 +0000 Subject: [PATCH 27/41] migrated remaining functionality --- pyo3-macros-backend/src/pyclass.rs | 6 +- src/exceptions.rs | 3 +- src/impl_/pycell.rs | 5 +- src/impl_/pyclass.rs | 22 +- src/impl_/pymethods.rs | 6 +- src/instance.rs | 15 - src/pycell.rs | 8 +- src/pycell/impl_.rs | 379 +++++++++++----------- src/type_object.rs | 18 +- src/types/any.rs | 7 +- src/types/ellipsis.rs | 9 +- src/types/mod.rs | 24 +- src/types/none.rs | 4 - src/types/notimplemented.rs | 9 +- src/types/typeobject.rs | 4 +- tests/test_variable_sized_class_basics.rs | 12 +- tests/ui/invalid_base_class.stderr | 20 +- 17 files changed, 256 insertions(+), 295 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index e0a9f083a07..d5a71a91cbb 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1813,8 +1813,6 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre const MODULE: ::std::option::Option<&'static str> = #module; const OPAQUE: bool = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE; - type Layout = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType as #pyo3_path::type_object::PyTypeInfo>::Layout; - #[inline] fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { use #pyo3_path::prelude::PyTypeMethods; @@ -2304,8 +2302,9 @@ impl<'a> PyClassImplsBuilder<'a> { let pyclass_base_type_impl = attr.options.subclass.map(|subclass| { quote_spanned! { subclass.span() => impl #pyo3_path::impl_::pyclass::PyClassBaseType for #cls { - type LayoutAsBase = ::Layout; + type StaticLayout = #pyo3_path::impl_::pycell::PyStaticClassLayout; type BaseNativeType = ::BaseNativeType; + type RecursiveOperations = #pyo3_path::impl_::pycell::PyClassRecursiveOperations; type Initializer = #pyo3_path::pyclass_init::PyClassInitializer; type PyClassMutability = ::PyClassMutability; } @@ -2321,7 +2320,6 @@ impl<'a> PyClassImplsBuilder<'a> { const IS_MAPPING: bool = #is_mapping; const IS_SEQUENCE: bool = #is_sequence; - type Layout = <#cls as #pyo3_path::PyTypeInfo>::Layout; type BaseType = #base; type ThreadChecker = #thread_checker; #inventory diff --git a/src/exceptions.rs b/src/exceptions.rs index 03ffc30fcf3..b682d7a0563 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -125,8 +125,7 @@ macro_rules! import_exception_bound { $name, $name::type_object_raw, ::std::option::Option::Some(stringify!($module)), - false, - $crate::impl_::pycell::PyStaticClassObject + false ); impl $crate::types::DerefToPyAny for $name {} diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index 2d1ae7268da..4c9e09b82c1 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,5 +1,6 @@ //! Externally-accessible implementation of pycell pub use crate::pycell::impl_::{ - opaque_layout::PyVariableClassObjectBase, static_layout::PyClassObjectBase, GetBorrowChecker, - PyClassMutability, PyClassObjectBaseLayout, PyStaticClassObject, PyVariableClassObject, + static_layout::InvalidStaticLayout, static_layout::PyStaticClassLayout, + static_layout::PyStaticNativeLayout, GetBorrowChecker, PyClassMutability, + PyClassRecursiveOperations, PyNativeTypeRecursiveOperations, PyObjectRecursiveOperations, }; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index b78c1c6895d..93928bba3f9 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -4,14 +4,15 @@ use crate::{ ffi, impl_::{ freelist::FreeList, - pycell::{GetBorrowChecker, PyClassMutability, PyClassObjectBaseLayout}, + pycell::{GetBorrowChecker, PyClassMutability}, pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, pycell::{ - impl_::{PyClassObjectLayout, PyObjectLayout}, + impl_::{PyObjectLayout, PyObjectRecursiveOperations}, PyBorrowError, }, + type_object::PyLayout, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, }; @@ -170,9 +171,6 @@ pub trait PyClassImpl: Sized + 'static { /// `#[pyclass(sequence)]` const IS_SEQUENCE: bool = false; - /// Description of how this class is laid out in memory - type Layout: PyClassObjectLayout; - /// Base class (the direct parent configured via `#[pyclass(extends=...)]`) type BaseType: PyTypeInfo + PyClassBaseType; @@ -1139,24 +1137,24 @@ impl PyClassThreadChecker for ThreadCheckerImpl { ) )] pub trait PyClassBaseType: Sized { - type LayoutAsBase: PyClassObjectBaseLayout; + /// A struct that describes the memory layout of a `*mut ffi:PyObject` with the type of `Self`. + /// Only valid when `::OPAQUE` is false. + type StaticLayout: PyLayout; + // TODO(matt): introduce :PyTypeInfo bounds type BaseNativeType; + type RecursiveOperations: PyObjectRecursiveOperations; type Initializer: PyObjectInit; type PyClassMutability: PyClassMutability; } /// Implementation of tp_dealloc for pyclasses without gc pub(crate) unsafe extern "C" fn tp_dealloc(obj: *mut ffi::PyObject) { - crate::impl_::trampoline::dealloc(obj, ::Layout::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, PyObjectLayout::deallocate::) } /// Implementation of tp_dealloc for pyclasses with gc pub(crate) unsafe extern "C" fn tp_dealloc_with_gc(obj: *mut ffi::PyObject) { - #[cfg(not(PyPy))] - { - ffi::PyObject_GC_UnTrack(obj.cast()); - } - crate::impl_::trampoline::dealloc(obj, ::Layout::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, PyObjectLayout::deallocate_with_gc::) } pub(crate) unsafe extern "C" fn get_sequence_item_from_mapping( diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index ecb749cc38a..233e836b0c7 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -2,7 +2,6 @@ use crate::exceptions::PyStopAsyncIteration; use crate::gil::LockGIL; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; -use crate::impl_::pycell::PyClassObjectBaseLayout; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; use crate::pycell::impl_::{PyClassBorrowChecker as _, PyObjectLayout}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; @@ -300,15 +299,14 @@ where return super_retval; } - let raw_obj = &*slf; // SAFETY: `slf` is a valid Python object pointer to a class object of type T, and // traversal is running so no mutations can occur. - let class_object: &::Layout = &*slf.cast(); + let raw_obj = &*slf; let retval = // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so // do not traverse them if not on their owning thread :( - if class_object.check_threadsafe().is_ok() + if PyObjectLayout::check_threadsafe::(raw_obj).is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread && PyObjectLayout::get_borrow_checker::(raw_obj).try_borrow().is_ok() { struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); diff --git a/src/instance.rs b/src/instance.rs index 890fdd0dd84..e9c3318c9a0 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,6 +1,5 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; -use crate::impl_::pyclass::PyClassImpl; use crate::internal_tricks::ptr_from_ref; use crate::pycell::impl_::PyObjectLayout; use crate::pycell::{PyBorrowError, PyBorrowMutError}; @@ -462,11 +461,6 @@ where unsafe { self.into_any().downcast_into_unchecked() } } - #[inline] - pub(crate) fn get_class_object(&self) -> &::Layout { - self.1.get_class_object() - } - #[inline] pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { self.1.get_raw_object() @@ -1300,15 +1294,6 @@ where unsafe { &*PyObjectLayout::get_data_ptr::(obj) } } - /// Get a view on the underlying `PyClass` contents. - #[inline] - pub(crate) fn get_class_object(&self) -> &::Layout { - let class_object = self.as_ptr().cast::<::Layout>(); - // Safety: Bound is known to contain an object which is laid out in memory as a - // ::Layout object - unsafe { &*class_object } - } - /// Get a reference to the underlying `PyObject` #[inline] pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { diff --git a/src/pycell.rs b/src/pycell.rs index 095ad6d2d4d..00f09c05a40 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -208,7 +208,7 @@ use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::{PyClassBorrowChecker, PyClassObjectBaseLayout, PyObjectLayout}; +use impl_::{PyClassBorrowChecker, PyObjectLayout}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// @@ -308,9 +308,8 @@ impl<'py, T: PyClass> PyRef<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_class_object(); - cell.ensure_threadsafe(); let raw_obj = obj.get_raw_object(); + unsafe { PyObjectLayout::ensure_threadsafe::(raw_obj) }; let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; borrow_checker .try_borrow() @@ -566,9 +565,8 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_class_object(); - cell.ensure_threadsafe(); let raw_obj = obj.get_raw_object(); + unsafe { PyObjectLayout::ensure_threadsafe::(raw_obj) }; let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; borrow_checker .try_borrow_mut() diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 21a06018caf..222679796dd 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -11,9 +11,9 @@ use crate::impl_::pyclass::{ PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, }; use crate::internal::get_slot::TP_FREE; -use crate::type_object::{PyLayout, PySizedLayout}; +use crate::type_object::PyNativeType; use crate::types::PyType; -use crate::{ffi, PyClass, PyTypeInfo, Python}; +use crate::{ffi, PyTypeInfo, Python}; #[cfg(not(Py_LIMITED_API))] use crate::types::PyTypeMethods; @@ -211,66 +211,6 @@ where } } -/// functionality common to all PyObjects regardless of the layout -#[doc(hidden)] -pub trait PyClassObjectBaseLayout: PyLayout { - fn ensure_threadsafe(&self); - fn check_threadsafe(&self) -> Result<(), PyBorrowError>; - /// Implementation of tp_dealloc. - /// # Safety - /// - slf must be a valid pointer to an instance of a T or a subclass. - /// - slf must not be used after this call (as it will be freed). - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); -} - -/// Functionality required for creating and managing the memory associated with a pyclass annotated struct. -#[doc(hidden)] -pub trait PyClassObjectLayout: PyClassObjectBaseLayout {} - -/// Implementation of `tp_dealloc`. -/// # Safety -/// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. -/// - obj must not be used after this call (as it will be freed). -unsafe fn tp_dealloc(py: Python<'_>, obj: *mut ffi::PyObject, type_ptr: *mut ffi::PyTypeObject) { - // FIXME: there is potentially subtle issues here if the base is overwritten - // at runtime? To be investigated. - let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); - - // TODO(matt): is this correct? - // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free - let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); - let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); - if is_base_object || is_metaclass { - let tp_free = actual_type - .get_slot(TP_FREE) - .expect("base type should have tp_free"); - return tp_free(obj.cast()); - } - - // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. - #[cfg(not(Py_LIMITED_API))] - { - // FIXME: should this be using actual_type.tp_dealloc? - if let Some(dealloc) = (*type_ptr).tp_dealloc { - // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which - // assumes the exception is currently GC tracked, so we have to re-track - // before calling the dealloc so that it can safely call Py_GC_UNTRACK. - #[cfg(not(any(Py_3_11, PyPy)))] - if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { - ffi::PyObject_GC_Track(obj.cast()); - } - dealloc(obj); - } else { - (*actual_type.as_type_ptr()) - .tp_free - .expect("type missing tp_free")(obj.cast()); - } - } - - #[cfg(Py_LIMITED_API)] - unreachable!("subclassing native types is not possible with the `abi3` feature"); -} - #[repr(C)] pub(crate) struct PyClassObjectContents { pub(crate) value: ManuallyDrop>, @@ -300,115 +240,127 @@ impl PyClassObjectContents { } } -/// The layout of a PyClassObject with a known sized base class. -#[repr(C)] -pub struct PyStaticClassObject { - ob_base: ::LayoutAsBase, - contents: PyClassObjectContents, -} - -impl PyClassObjectLayout for PyStaticClassObject {} - -unsafe impl PyLayout for PyStaticClassObject {} -impl PySizedLayout for PyStaticClassObject {} - -impl PyClassObjectBaseLayout for PyStaticClassObject -where - ::LayoutAsBase: PyClassObjectBaseLayout, -{ - fn ensure_threadsafe(&self) { - self.contents.thread_checker.ensure(); - self.ob_base.ensure_threadsafe(); - } - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - if !self.contents.thread_checker.check() { - return Err(PyBorrowError { _private: () }); - } - self.ob_base.check_threadsafe() - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - // Safety: Python only calls tp_dealloc when no references to the object remain. - let contents = unsafe { PyObjectLayout::get_contents_ptr::(slf) }; - (*contents).dealloc(py, slf); - ::LayoutAsBase::tp_dealloc(py, slf) - } +/// Py_ssize_t may not be equal to isize on all platforms +fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { + #[allow(clippy::useless_conversion)] + value.try_into().expect("value should fit in Py_ssize_t") } -/// A layout for a PyClassObject with an unknown sized base type. -/// -/// Utilises [PEP-697](https://peps.python.org/pep-0697/) +/// Functions for working with `PyObjects` recursively by re-interpreting the object +/// as being an instance of the most derived class through each base class until +/// the `BaseNativeType` is reached. #[doc(hidden)] -#[repr(C)] -pub struct PyVariableClassObject { - ob_base: ::LayoutAsBase, +pub trait PyObjectRecursiveOperations { + unsafe fn ensure_threadsafe(obj: *mut ffi::PyObject); + unsafe fn check_threadsafe(obj: *mut ffi::PyObject) -> Result<(), PyBorrowError>; + /// Cleanup then free the memory for `obj`. + /// + /// # Safety + /// - slf must be a valid pointer to an instance of a T or a subclass. + /// - slf must not be used after this call (as it will be freed). + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject); } -#[cfg(Py_3_12)] -impl PyClassObjectLayout for PyVariableClassObject {} - -unsafe impl PyLayout for PyVariableClassObject {} +/// Used to fill out `PyClassBaseType::RecursiveOperations` for instances of `PyClass` +pub struct PyClassRecursiveOperations(PhantomData); -#[cfg(Py_3_12)] -impl PyClassObjectBaseLayout for PyVariableClassObject -where - ::LayoutAsBase: PyClassObjectBaseLayout, -{ - fn ensure_threadsafe(&self) { - let obj_ptr = self as *const Self as *mut ffi::PyObject; - let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj_ptr) }; +impl PyObjectRecursiveOperations for PyClassRecursiveOperations { + unsafe fn ensure_threadsafe(obj: *mut ffi::PyObject) { + let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj) }; contents.thread_checker.ensure(); - self.ob_base.ensure_threadsafe(); + ::RecursiveOperations::ensure_threadsafe(obj); } - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - let obj_ptr = self as *const Self as *mut ffi::PyObject; - let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj_ptr) }; + + unsafe fn check_threadsafe(obj: *mut ffi::PyObject) -> Result<(), PyBorrowError> { + let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj) }; if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } - self.ob_base.check_threadsafe() + ::RecursiveOperations::check_threadsafe(obj) } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. - let contents = unsafe { PyObjectLayout::get_contents_ptr::(slf) }; - (*contents).dealloc(py, slf); - ::LayoutAsBase::tp_dealloc(py, slf) + let contents = unsafe { &mut *PyObjectLayout::get_contents_ptr::(obj) }; + contents.dealloc(py, obj); + ::RecursiveOperations::deallocate(py, obj); } } -/// Py_ssize_t may not be equal to isize on all platforms -fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { - #[allow(clippy::useless_conversion)] - value.try_into().expect("value should fit in Py_ssize_t") -} +/// Used to fill out `PyClassBaseType::RecursiveOperations` for native types +pub struct PyNativeTypeRecursiveOperations(PhantomData); -/// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). -#[doc(hidden)] -pub(crate) mod opaque_layout { - use super::{tp_dealloc, PyClassObjectBaseLayout, PyClassObjectContents}; - use crate::impl_::pyclass::PyClassImpl; - use crate::pycell::PyBorrowError; - use crate::type_object::PyLayout; - use crate::{ffi, PyTypeInfo, Python}; +impl PyObjectRecursiveOperations + for PyNativeTypeRecursiveOperations +{ + unsafe fn ensure_threadsafe(_obj: *mut ffi::PyObject) {} - /// Base layout of `PyClassObject` with an unknown sized base type. - /// Corresponds to [PyVarObject](https://docs.python.org/3/c-api/structures.html#c.PyVarObject) from the C API. - #[doc(hidden)] - #[repr(C)] - pub struct PyVariableClassObjectBase { - ob_base: ffi::PyVarObject, + unsafe fn check_threadsafe(_obj: *mut ffi::PyObject) -> Result<(), PyBorrowError> { + Ok(()) } - unsafe impl PyLayout for PyVariableClassObjectBase {} - - impl PyClassObjectBaseLayout for PyVariableClassObjectBase { - fn ensure_threadsafe(&self) {} - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - Ok(()) + /// Call the destructor (`tp_dealloc`) of an object which is an instance of a + /// subclass of the native type `T`. + /// + /// Does not clear up any data from subtypes of `type_ptr` so it is assumed that those + /// destructors have been called first. + /// + /// [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + /// + /// # Safety + /// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. + /// - obj must not be used after this call (as it will be freed). + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + // the `BaseNativeType` of the object + let type_ptr = ::type_object_raw(py); + + // FIXME: there is potentially subtle issues here if the base is overwritten at runtime? To be investigated. + + // the 'most derived class' of `obj`. i.e. the result of calling `type(obj)`. + let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); + + // TODO(matt): is this correct? + // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free + let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); + let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); + if is_base_object || is_metaclass { + let tp_free = actual_type + .get_slot(TP_FREE) + .expect("base type should have tp_free"); + return tp_free(obj.cast()); } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - tp_dealloc(py, slf, T::type_object_raw(py)); + + // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. + #[cfg(not(Py_LIMITED_API))] + { + // FIXME: should this be using actual_type.tp_dealloc? + if let Some(dealloc) = (*type_ptr).tp_dealloc { + // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which + // assumes the exception is currently GC tracked, so we have to re-track + // before calling the dealloc so that it can safely call Py_GC_UNTRACK. + #[cfg(not(any(Py_3_11, PyPy)))] + if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { + ffi::PyObject_GC_Track(obj.cast()); + } + dealloc(obj); + } else { + (*actual_type.as_type_ptr()) + .tp_free + .expect("type missing tp_free")(obj.cast()); + } } + + #[cfg(Py_LIMITED_API)] + unreachable!("subclassing native types is not possible with the `abi3` feature"); } +} + +/// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). +#[doc(hidden)] +pub(crate) mod opaque_layout { + use super::PyClassObjectContents; + use crate::ffi; + use crate::impl_::pyclass::PyClassImpl; #[cfg(Py_3_12)] pub fn get_contents_ptr( @@ -440,45 +392,39 @@ pub(crate) mod opaque_layout { #[doc(hidden)] pub(crate) mod static_layout { use crate::{ - ffi, impl_::pyclass::{PyClassBaseType, PyClassImpl}, - pycell::PyBorrowError, type_object::{PyLayout, PySizedLayout}, - PyTypeInfo, Python, }; - use super::{tp_dealloc, PyClassObjectBaseLayout, PyClassObjectContents}; + use super::PyClassObjectContents; // The layout of a `PyObject` that uses the static layout #[repr(C)] - pub struct ClassObject { - pub ob_base: ::LayoutAsBase, - pub contents: PyClassObjectContents, + pub struct PyStaticClassLayout { + pub(crate) ob_base: ::StaticLayout, + pub(crate) contents: PyClassObjectContents, } + unsafe impl PyLayout for PyStaticClassLayout {} + /// Base layout of PyClassObject with a known sized base type. /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. #[doc(hidden)] #[repr(C)] - pub struct PyClassObjectBase { + pub struct PyStaticNativeLayout { ob_base: T, } - unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} + unsafe impl PyLayout for PyStaticNativeLayout where U: PySizedLayout {} - impl PyClassObjectBaseLayout for PyClassObjectBase - where - U: PySizedLayout, - T: PyTypeInfo, - { - fn ensure_threadsafe(&self) {} - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - Ok(()) - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - tp_dealloc(py, slf, T::type_object_raw(py)); - } - } + /// a struct for use with opaque native types to indicate that they + /// cannot be used as part of a static layout. + #[repr(C)] + pub struct InvalidStaticLayout; + + /// This is valid insofar as casting a `*mut ffi::PyObject` to `*mut InvalidStaticLayout` is valid + /// since nothing can actually be read by dereferencing. + unsafe impl PyLayout for InvalidStaticLayout {} } /// Functions for working with `PyObject`s @@ -495,7 +441,13 @@ impl PyObjectLayout { if ::OPAQUE { opaque_layout::get_contents_ptr(obj) } else { - let obj: *mut static_layout::ClassObject = obj.cast(); + let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); + // indicates `ob_base` has type InvalidBaseLayout + debug_assert_ne!( + std::mem::offset_of!(static_layout::PyStaticClassLayout, contents), + 0, + "invalid ob_base found" + ); addr_of_mut!((*obj).contents) } } @@ -520,14 +472,53 @@ impl PyObjectLayout { T::PyClassMutability::borrow_checker(obj) } - /// obtain a reference to the data at the start of the `PyObject`. + pub(crate) unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { + unsafe { + PyClassRecursiveOperations::::ensure_threadsafe( + obj as *const ffi::PyObject as *mut ffi::PyObject, + ) + }; + } + + pub(crate) unsafe fn check_threadsafe( + obj: &ffi::PyObject, + ) -> Result<(), PyBorrowError> { + unsafe { + PyClassRecursiveOperations::::check_threadsafe( + obj as *const ffi::PyObject as *mut ffi::PyObject, + ) + } + } + + /// Clean up then free the memory associated with `obj`. /// - /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn ob_base( - obj: *mut ffi::PyObject, - ) -> *mut ::LayoutAsBase { - // the base layout is always at the beginning of the `PyObject` so the pointer can simply be casted - obj as *mut ::LayoutAsBase + /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + unsafe { + PyClassRecursiveOperations::::deallocate( + py, + obj as *const ffi::PyObject as *mut ffi::PyObject, + ) + }; + } + + /// Clean up then free the memory associated with `obj`. + /// + /// Use instead of `deallocate()` if `T` has the `Py_TPFLAGS_HAVE_GC` flag set. + /// + /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + pub(crate) fn deallocate_with_gc(py: Python<'_>, obj: *mut ffi::PyObject) { + unsafe { + // TODO(matt): verify T has flag set + #[cfg(not(PyPy))] + { + ffi::PyObject_GC_UnTrack(obj.cast()); + } + PyClassRecursiveOperations::::deallocate( + py, + obj as *const ffi::PyObject as *mut ffi::PyObject, + ) + }; } /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` @@ -544,7 +535,7 @@ impl PyObjectLayout { #[cfg(not(Py_3_12))] opaque_layout::panic_unsupported(); } else { - usize_to_py_ssize(std::mem::size_of::>()) + usize_to_py_ssize(std::mem::size_of::>()) } } @@ -560,7 +551,7 @@ impl PyObjectLayout { opaque_layout::panic_unsupported(); } else { PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( - static_layout::ClassObject, + static_layout::PyStaticClassLayout, contents ))) } @@ -580,7 +571,7 @@ impl PyObjectLayout { #[cfg(not(Py_3_12))] opaque_layout::panic_unsupported(); } else { - let offset = memoffset::offset_of!(static_layout::ClassObject, contents) + let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + memoffset::offset_of!(PyClassObjectContents, dict); PyObjectOffset::Absolute(usize_to_py_ssize(offset)) @@ -601,7 +592,7 @@ impl PyObjectLayout { #[cfg(not(Py_3_12))] opaque_layout::panic_unsupported(); } else { - let offset = memoffset::offset_of!(static_layout::ClassObject, contents) + let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + memoffset::offset_of!(PyClassObjectContents, weakref); PyObjectOffset::Absolute(usize_to_py_ssize(offset)) @@ -628,15 +619,6 @@ impl<'a, T: PyClassImpl> PyObjectHandle<'a, T> { pub fn data(&'a self) -> &'a T { unsafe { &*PyObjectLayout::get_data_ptr::(self.0) } } - - pub fn ensure_threadsafe(&self) { - let contents = unsafe { &(*PyObjectLayout::get_contents_ptr::(self.0)) }; - contents.thread_checker.ensure(); - let base = unsafe { &(*PyObjectLayout::ob_base::(self.0)) }; - base.ensure_threadsafe(); - } - - pub fn borrow_checker(&self) {} } #[cfg(test)] @@ -644,8 +626,8 @@ impl<'a, T: PyClassImpl> PyObjectHandle<'a, T> { mod tests { use super::*; - use crate::prelude::*; use crate::pyclass::boolean_struct::{False, True}; + use crate::{prelude::*, PyClass}; #[pyclass(crate = "crate", subclass)] struct MutableBase; @@ -706,6 +688,19 @@ mod tests { assert!(base_size < PyObjectLayout::basicsize::()); } + #[test] + fn test_invalid_base() { + assert_eq!(std::mem::size_of::(), 0); + + #[repr(C)] + struct InvalidLayout { + ob_base: static_layout::InvalidStaticLayout, + contents: u8, + } + + assert_eq!(std::mem::offset_of!(InvalidLayout, contents), 0); + } + fn assert_mutable>() {} fn assert_immutable>() {} fn assert_mutable_with_mutable_ancestor< diff --git a/src/type_object.rs b/src/type_object.rs index e7e366cb3f4..46b8c32e91e 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,14 +1,22 @@ //! Python type object information use crate::ffi_ptr_ext::FfiPtrExt; -use crate::impl_::pyclass::PyClassImpl; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyType}; use crate::{ffi, Bound, Python}; +/// `T: PyNativeType` represents that `T` is a struct representing a 'native python class'. +/// a 'native class' is a wrapper around a `ffi::PyTypeObject` that is defined by the python +/// API such as `PyDict` for `dict`. +/// +/// This trait is intended to be used internally. +/// +/// # Safety +/// +/// This trait must only be implemented for types which represent native python classes. +pub unsafe trait PyNativeType {} + /// `T: PyLayout` represents that `T` is a concrete representation of `U` in the Python heap. -/// E.g., `PyClassObject` is a concrete representation of all `pyclass`es, and `ffi::PyObject` -/// is of `PyAny`. /// /// This trait is intended to be used internally. /// @@ -48,10 +56,6 @@ pub unsafe trait PyTypeInfo: Sized { /// of a new `repr(C)` struct const OPAQUE: bool; - /// The type of object layout to use for ancestors or descendants of this type. - /// should implement `PyClassObjectLayout` in order to actually use it as a layout. - type Layout; - /// Returns the `PyTypeObject` instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; diff --git a/src/types/any.rs b/src/types/any.rs index 842e12d95e9..492929fe4cd 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -3,7 +3,6 @@ use crate::conversion::{AsPyPointer, FromPyObjectBound, IntoPyObject}; use crate::err::{DowncastError, DowncastIntoError, PyErr, PyResult}; use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::ffi_ptr_ext::FfiPtrExt; -use crate::impl_::pycell::PyStaticClassObject; use crate::instance::Bound; use crate::internal::get_slot::TP_DESCR_GET; use crate::internal_tricks::ptr_from_ref; @@ -43,15 +42,15 @@ pyobject_native_type_info!( pyobject_native_static_type_object!(ffi::PyBaseObject_Type), Some("builtins"), false, - PyStaticClassObject, #checkfunction=PyObject_Check ); - +pyobject_native_type_marker!(PyAny); pyobject_native_type_sized!(PyAny, ffi::PyObject); // We cannot use `pyobject_subclassable_native_type!()` because it cfgs out on `Py_LIMITED_API`. impl crate::impl_::pyclass::PyClassBaseType for PyAny { - type LayoutAsBase = crate::impl_::pycell::PyClassObjectBase; + type StaticLayout = crate::impl_::pycell::PyStaticNativeLayout; type BaseNativeType = PyAny; + type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; type PyClassMutability = crate::pycell::impl_::ImmutableClass; } diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index 9ea2f2a4030..959123ea9db 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -1,9 +1,6 @@ use crate::{ - ffi, - ffi_ptr_ext::FfiPtrExt, - impl_::{pycell::PyStaticClassObject, pyclass::PyClassImpl}, - types::any::PyAnyMethods, - Borrowed, Bound, PyAny, PyTypeInfo, Python, + ffi, ffi_ptr_ext::FfiPtrExt, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyTypeInfo, + Python, }; /// Represents the Python `Ellipsis` object. @@ -35,8 +32,6 @@ unsafe impl PyTypeInfo for PyEllipsis { const MODULE: Option<&'static str> = None; const OPAQUE: bool = false; - type Layout = PyStaticClassObject; - fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_Ellipsis()) } } diff --git a/src/types/mod.rs b/src/types/mod.rs index f35d7cd0cfa..07fa0578d49 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -153,14 +153,12 @@ macro_rules! pyobject_native_static_type_object( #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_info( - ($name:ty, $typeobject:expr, $module:expr, $opaque:expr, $layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $typeobject:expr, $module:expr, $opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { unsafe impl<$($generics,)*> $crate::type_object::PyTypeInfo for $name { const NAME: &'static str = stringify!($name); const MODULE: ::std::option::Option<&'static str> = $module; const OPAQUE: bool = $opaque; - type Layout = $layout; - #[inline] #[allow(clippy::redundant_closure_call)] fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { @@ -183,19 +181,28 @@ macro_rules! pyobject_native_type_info( }; ); +#[doc(hidden)] +#[macro_export] +macro_rules! pyobject_native_type_marker( + ($name:ty) => { + unsafe impl $crate::type_object::PyNativeType for $name {} + } +); + /// Declares all of the boilerplate for Python types. #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_core { - ($name:ty, $typeobject:expr, #module=$module:expr, #opaque=$opaque:expr, #layout=$layout:path $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $typeobject:expr, #module=$module:expr, #opaque=$opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { $crate::pyobject_native_type_named!($name $(;$generics)*); - $crate::pyobject_native_type_info!($name, $typeobject, $module, $opaque, $layout $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_marker!($name); + $crate::pyobject_native_type_info!($name, $typeobject, $module, $opaque $(, #checkfunction=$checkfunction)? $(;$generics)*); }; ($name:ty, $typeobject:expr, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=$module, #opaque=false, #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_core!($name, $typeobject, #module=$module, #opaque=false $(, #checkfunction=$checkfunction)? $(;$generics)*); }; ($name:ty, $typeobject:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins"), #opaque=false, #layout=$crate::impl_::pycell::PyStaticClassObject $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins"), #opaque=false $(, #checkfunction=$checkfunction)? $(;$generics)*); }; } @@ -205,8 +212,9 @@ macro_rules! pyobject_subclassable_native_type { ($name:ty, $layout:path $(;$generics:ident)*) => { #[cfg(not(Py_LIMITED_API))] impl<$($generics,)*> $crate::impl_::pyclass::PyClassBaseType for $name { - type LayoutAsBase = $crate::impl_::pycell::PyClassObjectBase<$layout>; + type StaticLayout = $crate::impl_::pycell::PyStaticNativeLayout<$layout>; type BaseNativeType = $name; + type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = $crate::impl_::pyclass_init::PyNativeTypeInitializer; type PyClassMutability = $crate::pycell::impl_::ImmutableClass; } diff --git a/src/types/none.rs b/src/types/none.rs index 86e80bbe2e9..14eb050a94c 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -1,6 +1,4 @@ use crate::ffi_ptr_ext::FfiPtrExt; -use crate::impl_::pycell::PyStaticClassObject; -use crate::impl_::pyclass::PyClassImpl; use crate::{ffi, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyObject, PyTypeInfo, Python}; #[allow(deprecated)] use crate::{IntoPy, ToPyObject}; @@ -34,8 +32,6 @@ unsafe impl PyTypeInfo for PyNone { const MODULE: Option<&'static str> = None; const OPAQUE: bool = false; - type Layout = PyStaticClassObject; - fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_None()) } } diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index fe4e4556cb3..e8f1dd94ef0 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -1,9 +1,6 @@ use crate::{ - ffi, - ffi_ptr_ext::FfiPtrExt, - impl_::{pycell::PyStaticClassObject, pyclass::PyClassImpl}, - types::any::PyAnyMethods, - Borrowed, Bound, PyAny, PyTypeInfo, Python, + ffi, ffi_ptr_ext::FfiPtrExt, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyTypeInfo, + Python, }; /// Represents the Python `NotImplemented` object. @@ -39,8 +36,6 @@ unsafe impl PyTypeInfo for PyNotImplemented { const MODULE: Option<&'static str> = None; const OPAQUE: bool = false; - type Layout = PyStaticClassObject; - fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_NotImplemented()) } } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index a0da756c411..4744a003339 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -22,13 +22,13 @@ pyobject_native_type_core!( pyobject_native_static_type_object!(ffi::PyType_Type), #module=::std::option::Option::Some("builtins"), #opaque=true, - #layout=crate::impl_::pycell::PyVariableClassObject, #checkfunction = ffi::PyType_Check ); impl crate::impl_::pyclass::PyClassBaseType for PyType { - type LayoutAsBase = crate::impl_::pycell::PyVariableClassObjectBase; + type StaticLayout = crate::impl_::pycell::InvalidStaticLayout; type BaseNativeType = PyType; + type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; type PyClassMutability = crate::pycell::impl_::ImmutableClass; } diff --git a/tests/test_variable_sized_class_basics.rs b/tests/test_variable_sized_class_basics.rs index 5b141e8c9ac..6b8e2c04579 100644 --- a/tests/test_variable_sized_class_basics.rs +++ b/tests/test_variable_sized_class_basics.rs @@ -1,20 +1,12 @@ #![cfg(all(Py_3_12, feature = "macros"))] -use std::any::TypeId; - -use pyo3::impl_::pycell::PyVariableClassObject; -use pyo3::impl_::pyclass::PyClassImpl; -use pyo3::py_run; use pyo3::types::{PyDict, PyInt, PyTuple}; use pyo3::{prelude::*, types::PyType}; +use pyo3::{py_run, PyTypeInfo}; #[path = "../src/tests/common.rs"] mod common; -fn uses_variable_layout() -> bool { - TypeId::of::() == TypeId::of::>() -} - #[pyclass(extends=PyType)] #[derive(Default)] struct ClassWithObjectField { @@ -37,7 +29,7 @@ impl ClassWithObjectField { fn class_with_object_field() { Python::with_gil(|py| { let ty = py.get_type::(); - assert!(uses_variable_layout::()); + assert!(::OPAQUE); py_run!( py, ty, diff --git a/tests/ui/invalid_base_class.stderr b/tests/ui/invalid_base_class.stderr index a9262564cec..c40bed9eaa6 100644 --- a/tests/ui/invalid_base_class.stderr +++ b/tests/ui/invalid_base_class.stderr @@ -1,8 +1,8 @@ error[E0277]: pyclass `PyBool` cannot be subclassed - --> tests/ui/invalid_base_class.rs:4:1 + --> tests/ui/invalid_base_class.rs:4:19 | 4 | #[pyclass(extends=PyBool)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyBool)]` + | ^^^^^^ required for `#[pyclass(extends=PyBool)]` | = help: the trait `PyClassBaseType` is not implemented for `PyBool` = note: `PyBool` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -16,13 +16,17 @@ error[E0277]: pyclass `PyBool` cannot be subclassed PyBlockingIOError PyBrokenPipeError and $N others - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` error[E0277]: pyclass `PyBool` cannot be subclassed - --> tests/ui/invalid_base_class.rs:4:19 + --> tests/ui/invalid_base_class.rs:4:1 | 4 | #[pyclass(extends=PyBool)] - | ^^^^^^ required for `#[pyclass(extends=PyBool)]` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyBool)]` | = help: the trait `PyClassBaseType` is not implemented for `PyBool` = note: `PyBool` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -36,8 +40,4 @@ error[E0277]: pyclass `PyBool` cannot be subclassed PyBlockingIOError PyBrokenPipeError and $N others -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) From 837743a3919db7b9de23dd4eba2c59d51f4f2b3f Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 9 Nov 2024 15:59:41 +0000 Subject: [PATCH 28/41] simplify interface by using references where possible --- src/impl_/pyclass.rs | 2 +- src/impl_/pymethods.rs | 4 +- src/instance.rs | 36 ++++++---- src/pycell.rs | 15 ++-- src/pycell/impl_.rs | 83 +++++++---------------- src/types/mod.rs | 3 +- tests/test_variable_sized_class_basics.rs | 3 +- 7 files changed, 62 insertions(+), 84 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 93928bba3f9..f56f05c7609 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1546,7 +1546,7 @@ where PyObjectOffset::Absolute(offset) => (obj.cast::(), offset), #[cfg(Py_3_12)] PyObjectOffset::Relative(offset) => { - // Safety: obj must be a valid `PyObject` of type `ClassT` + // Safety: obj must be a valid `PyObject` whose type is a subtype of `ClassT` let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; (contents.cast::(), offset) } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 233e836b0c7..53882749f5c 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -312,7 +312,7 @@ where struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); impl Drop for TraverseGuard<'_, Cls> { fn drop(&mut self) { - let borrow_checker = unsafe {PyObjectLayout::get_borrow_checker::(self.0)}; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(self.0) }; borrow_checker.release_borrow(); } } @@ -320,7 +320,7 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); - let instance = unsafe {PyObjectLayout::get_data::(raw_obj)}; + let instance = PyObjectLayout::get_data::(raw_obj); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index e9c3318c9a0..fc20b626a3e 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -463,7 +463,7 @@ where #[inline] pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { - self.1.get_raw_object() + self.1.as_raw_ref() } } @@ -553,6 +553,17 @@ impl<'py, T> Bound<'py, T> { self.1.as_ptr() } + /// Returns the raw FFI object represented by self. + /// + /// # Safety + /// + /// The reference is borrowed; callers should not decrease the reference count + /// when they are finished with the object. + #[inline] + pub fn as_raw_ref(&self) -> &ffi::PyObject { + self.1.as_raw_ref() + } + /// Returns an owned raw FFI pointer represented by self. /// /// # Safety @@ -1116,6 +1127,17 @@ impl Py { self.0.as_ptr() } + /// Returns the raw FFI object represented by self. + /// + /// # Safety + /// + /// The reference is borrowed; callers should not decrease the reference count + /// when they are finished with the object. + #[inline] + pub fn as_raw_ref(&self) -> &ffi::PyObject { + unsafe { &*self.0.as_ptr() } + } + /// Returns an owned raw FFI pointer represented by self. /// /// # Safety @@ -1289,18 +1311,8 @@ where where T: PyClass + Sync, { - let obj = self.as_ptr(); // Safety: The class itself is frozen and `Sync` - unsafe { &*PyObjectLayout::get_data_ptr::(obj) } - } - - /// Get a reference to the underlying `PyObject` - #[inline] - pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { - let obj = self.as_ptr(); - assert!(!obj.is_null()); - // Safety: obj is a valid pointer to a PyObject - unsafe { &*obj } + unsafe { PyObjectLayout::get_data::(self.as_raw_ref()) } } } diff --git a/src/pycell.rs b/src/pycell.rs index 00f09c05a40..bb8a65b6563 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -308,7 +308,7 @@ impl<'py, T: PyClass> PyRef<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let raw_obj = obj.get_raw_object(); + let raw_obj = obj.as_raw_ref(); unsafe { PyObjectLayout::ensure_threadsafe::(raw_obj) }; let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; borrow_checker @@ -437,14 +437,13 @@ impl Deref for PyRef<'_, T> { #[inline] fn deref(&self) -> &T { - let obj = self.inner.as_ptr(); - unsafe { &*PyObjectLayout::get_data_ptr::(obj) } + unsafe { PyObjectLayout::get_data::(self.inner.as_raw_ref()) } } } impl Drop for PyRef<'_, T> { fn drop(&mut self) { - let obj = self.inner.get_raw_object(); + let obj = self.inner.as_raw_ref(); let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(obj) }; borrow_checker.release_borrow(); } @@ -565,7 +564,7 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let raw_obj = obj.get_raw_object(); + let raw_obj = obj.as_raw_ref(); unsafe { PyObjectLayout::ensure_threadsafe::(raw_obj) }; let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; borrow_checker @@ -622,16 +621,14 @@ impl> Deref for PyRefMut<'_, T> { #[inline] fn deref(&self) -> &T { - let obj = self.inner.as_ptr(); - unsafe { &*PyObjectLayout::get_data_ptr::(obj) } + unsafe { PyObjectLayout::get_data::(self.inner.as_raw_ref()) } } } impl> DerefMut for PyRefMut<'_, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - let obj = self.inner.as_ptr(); - unsafe { &mut *PyObjectLayout::get_data_ptr::(obj) } + unsafe { &mut *PyObjectLayout::get_data_ptr::(self.inner.as_ptr()) } } } diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 222679796dd..25b23ef5c80 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -18,7 +18,7 @@ use crate::{ffi, PyTypeInfo, Python}; #[cfg(not(Py_LIMITED_API))] use crate::types::PyTypeMethods; -use super::{PyBorrowError, PyBorrowMutError}; +use super::{ptr_from_ref, PyBorrowError, PyBorrowMutError}; pub trait PyClassMutability { // The storage for this inheritance layer. Only the first mutable class in @@ -184,17 +184,15 @@ pub trait GetBorrowChecker { impl> GetBorrowChecker for MutableClass { fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { - let obj = (obj as *const ffi::PyObject).cast_mut(); - let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; - unsafe { &(*contents).borrow_checker } + let contents = unsafe { PyObjectLayout::get_contents::(obj) }; + &contents.borrow_checker } } impl> GetBorrowChecker for ImmutableClass { fn borrow_checker(obj: &ffi::PyObject) -> &EmptySlot { - let obj = (obj as *const ffi::PyObject).cast_mut(); - let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; - unsafe { &(*contents).borrow_checker } + let contents = unsafe { PyObjectLayout::get_contents::(obj) }; + &contents.borrow_checker } } @@ -251,8 +249,8 @@ fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { /// the `BaseNativeType` is reached. #[doc(hidden)] pub trait PyObjectRecursiveOperations { - unsafe fn ensure_threadsafe(obj: *mut ffi::PyObject); - unsafe fn check_threadsafe(obj: *mut ffi::PyObject) -> Result<(), PyBorrowError>; + unsafe fn ensure_threadsafe(obj: &ffi::PyObject); + unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError>; /// Cleanup then free the memory for `obj`. /// /// # Safety @@ -265,14 +263,14 @@ pub trait PyObjectRecursiveOperations { pub struct PyClassRecursiveOperations(PhantomData); impl PyObjectRecursiveOperations for PyClassRecursiveOperations { - unsafe fn ensure_threadsafe(obj: *mut ffi::PyObject) { - let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj) }; + unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { + let contents = PyObjectLayout::get_contents::(obj); contents.thread_checker.ensure(); ::RecursiveOperations::ensure_threadsafe(obj); } - unsafe fn check_threadsafe(obj: *mut ffi::PyObject) -> Result<(), PyBorrowError> { - let contents = unsafe { &*PyObjectLayout::get_contents_ptr::(obj) }; + unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError> { + let contents = PyObjectLayout::get_contents::(obj); if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } @@ -281,7 +279,7 @@ impl PyObjectRecursiveOperations for PyClassRecursiveOperations< unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. - let contents = unsafe { &mut *PyObjectLayout::get_contents_ptr::(obj) }; + let contents = &mut *PyObjectLayout::get_contents_ptr::(obj); contents.dealloc(py, obj); ::RecursiveOperations::deallocate(py, obj); } @@ -293,9 +291,9 @@ pub struct PyNativeTypeRecursiveOperations(PhantomData); impl PyObjectRecursiveOperations for PyNativeTypeRecursiveOperations { - unsafe fn ensure_threadsafe(_obj: *mut ffi::PyObject) {} + unsafe fn ensure_threadsafe(_obj: &ffi::PyObject) {} - unsafe fn check_threadsafe(_obj: *mut ffi::PyObject) -> Result<(), PyBorrowError> { + unsafe fn check_threadsafe(_obj: &ffi::PyObject) -> Result<(), PyBorrowError> { Ok(()) } @@ -452,6 +450,12 @@ impl PyObjectLayout { } } + pub(crate) unsafe fn get_contents( + obj: &ffi::PyObject, + ) -> &PyClassObjectContents { + &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut()).cast_const() + } + /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` @@ -461,9 +465,7 @@ impl PyObjectLayout { } pub(crate) unsafe fn get_data(obj: &ffi::PyObject) -> &T { - let obj_ref = (obj as *const ffi::PyObject).cast_mut(); - let data_ptr = unsafe { PyObjectLayout::get_data_ptr::(obj_ref) }; - unsafe { &*data_ptr } + &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut()) } pub(crate) unsafe fn get_borrow_checker( @@ -473,21 +475,13 @@ impl PyObjectLayout { } pub(crate) unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { - unsafe { - PyClassRecursiveOperations::::ensure_threadsafe( - obj as *const ffi::PyObject as *mut ffi::PyObject, - ) - }; + PyClassRecursiveOperations::::ensure_threadsafe(obj) } pub(crate) unsafe fn check_threadsafe( obj: &ffi::PyObject, ) -> Result<(), PyBorrowError> { - unsafe { - PyClassRecursiveOperations::::check_threadsafe( - obj as *const ffi::PyObject as *mut ffi::PyObject, - ) - } + PyClassRecursiveOperations::::check_threadsafe(obj) } /// Clean up then free the memory associated with `obj`. @@ -495,10 +489,7 @@ impl PyObjectLayout { /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { unsafe { - PyClassRecursiveOperations::::deallocate( - py, - obj as *const ffi::PyObject as *mut ffi::PyObject, - ) + PyClassRecursiveOperations::::deallocate(py, obj); }; } @@ -514,10 +505,7 @@ impl PyObjectLayout { { ffi::PyObject_GC_UnTrack(obj.cast()); } - PyClassRecursiveOperations::::deallocate( - py, - obj as *const ffi::PyObject as *mut ffi::PyObject, - ) + PyClassRecursiveOperations::::deallocate(py, obj); }; } @@ -600,27 +588,6 @@ impl PyObjectLayout { } } -/// A wrapper around PyObject to provide [PyObjectLayout] functionality. -#[allow(unused)] -pub(crate) struct PyObjectHandle<'a, T: PyClassImpl>(*mut ffi::PyObject, PhantomData<&'a T>); - -#[allow(unused)] -impl<'a, T: PyClassImpl> PyObjectHandle<'a, T> { - /// Safety: obj must point to a valid PyObject with the type `T` - pub unsafe fn new(obj: *mut ffi::PyObject) -> Self { - debug_assert!(!obj.is_null()); - PyObjectHandle(obj, PhantomData) - } - - pub fn contents(&'a self) -> &'a PyClassObjectContents { - unsafe { &*PyObjectLayout::get_contents_ptr::(self.0) } - } - - pub fn data(&'a self) -> &'a T { - unsafe { &*PyObjectLayout::get_data_ptr::(self.0) } - } -} - #[cfg(test)] #[cfg(feature = "macros")] mod tests { diff --git a/src/types/mod.rs b/src/types/mod.rs index 07fa0578d49..58d8732ac8f 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -139,6 +139,7 @@ macro_rules! pyobject_native_type_named ( }; ); +/// Obtain the address of the given static `PyTypeObject`. #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_static_type_object( @@ -214,7 +215,7 @@ macro_rules! pyobject_subclassable_native_type { impl<$($generics,)*> $crate::impl_::pyclass::PyClassBaseType for $name { type StaticLayout = $crate::impl_::pycell::PyStaticNativeLayout<$layout>; type BaseNativeType = $name; - type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; + type RecursiveOperations = $crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = $crate::impl_::pyclass_init::PyNativeTypeInitializer; type PyClassMutability = $crate::pycell::impl_::ImmutableClass; } diff --git a/tests/test_variable_sized_class_basics.rs b/tests/test_variable_sized_class_basics.rs index 6b8e2c04579..e190e400ef6 100644 --- a/tests/test_variable_sized_class_basics.rs +++ b/tests/test_variable_sized_class_basics.rs @@ -3,6 +3,7 @@ use pyo3::types::{PyDict, PyInt, PyTuple}; use pyo3::{prelude::*, types::PyType}; use pyo3::{py_run, PyTypeInfo}; +use static_assertions::const_assert; #[path = "../src/tests/common.rs"] mod common; @@ -29,7 +30,7 @@ impl ClassWithObjectField { fn class_with_object_field() { Python::with_gil(|py| { let ty = py.get_type::(); - assert!(::OPAQUE); + const_assert!(::OPAQUE); py_run!( py, ty, From 58f9680a3e25432aaffd95b916756433b57d9944 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 9 Nov 2024 17:36:49 +0000 Subject: [PATCH 29/41] organised into two modules --- src/impl_/coroutine.rs | 3 +- src/impl_/pycell.rs | 7 +- src/impl_/pyclass.rs | 4 +- src/impl_/pyclass_init.rs | 2 +- src/impl_/pymethods.rs | 3 +- src/instance.rs | 3 +- src/pycell.rs | 6 +- src/pycell/{impl_.rs => borrow_checker.rs} | 419 +------------------ src/pycell/layout.rs | 451 +++++++++++++++++++++ src/pyclass/create_type_object.rs | 2 +- src/pyclass_init.rs | 4 +- src/types/any.rs | 2 +- src/types/mod.rs | 2 +- src/types/typeobject.rs | 2 +- 14 files changed, 478 insertions(+), 432 deletions(-) rename src/pycell/{impl_.rs => borrow_checker.rs} (52%) create mode 100644 src/pycell/layout.rs diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index 9856ba0b013..2c7924fffd2 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -6,7 +6,8 @@ use std::{ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, - pycell::impl_::{PyClassBorrowChecker, PyClassObjectLayout}, + pycell::borrow_checker::PyClassBorrowChecker, + pycell::layout::PyClassObjectLayout, pyclass::boolean_struct::False, types::{PyAnyMethods, PyString}, IntoPyObject, Py, PyAny, PyClass, PyErr, PyResult, Python, diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index 4c9e09b82c1..fe4378ddc1d 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,6 +1,7 @@ //! Externally-accessible implementation of pycell -pub use crate::pycell::impl_::{ +pub use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassMutability}; +pub use crate::pycell::layout::{ static_layout::InvalidStaticLayout, static_layout::PyStaticClassLayout, - static_layout::PyStaticNativeLayout, GetBorrowChecker, PyClassMutability, - PyClassRecursiveOperations, PyNativeTypeRecursiveOperations, PyObjectRecursiveOperations, + static_layout::PyStaticNativeLayout, PyClassRecursiveOperations, + PyNativeTypeRecursiveOperations, PyObjectRecursiveOperations, }; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index f56f05c7609..d51f93bce7e 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -9,7 +9,7 @@ use crate::{ pymethods::{PyGetterDef, PyMethodDefType}, }, pycell::{ - impl_::{PyObjectLayout, PyObjectRecursiveOperations}, + layout::{PyObjectLayout, PyObjectRecursiveOperations}, PyBorrowError, }, type_object::PyLayout, @@ -1637,7 +1637,7 @@ fn pyo3_get_value< #[cfg(test)] #[cfg(feature = "macros")] mod tests { - use crate::pycell::impl_::PyClassObjectContents; + use crate::pycell::layout::PyClassObjectContents; use super::*; diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 1727da3e53d..a9983515287 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -2,7 +2,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::PyClassImpl; use crate::internal::get_slot::TP_ALLOC; -use crate::pycell::impl_::{PyClassObjectContents, PyObjectLayout}; +use crate::pycell::layout::{PyClassObjectContents, PyObjectLayout}; use crate::types::PyType; use crate::{ffi, Borrowed, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 53882749f5c..66ae6948ad6 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -3,7 +3,8 @@ use crate::gil::LockGIL; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; -use crate::pycell::impl_::{PyClassBorrowChecker as _, PyObjectLayout}; +use crate::pycell::borrow_checker::PyClassBorrowChecker; +use crate::pycell::layout::PyObjectLayout; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; diff --git a/src/instance.rs b/src/instance.rs index fc20b626a3e..d23a5efd598 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,8 +1,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::internal_tricks::ptr_from_ref; -use crate::pycell::impl_::PyObjectLayout; -use crate::pycell::{PyBorrowError, PyBorrowMutError}; +use crate::pycell::{layout::PyObjectLayout, PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; use crate::types::{DerefToPyAny, PyDict, PyString, PyTuple}; diff --git a/src/pycell.rs b/src/pycell.rs index bb8a65b6563..c046592d279 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -207,8 +207,10 @@ use std::fmt; use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; -pub(crate) mod impl_; -use impl_::{PyClassBorrowChecker, PyObjectLayout}; +pub(crate) mod borrow_checker; +pub(crate) mod layout; +use borrow_checker::PyClassBorrowChecker; +use layout::PyObjectLayout; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// diff --git a/src/pycell/impl_.rs b/src/pycell/borrow_checker.rs similarity index 52% rename from src/pycell/impl_.rs rename to src/pycell/borrow_checker.rs index 25b23ef5c80..926ec609e36 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/borrow_checker.rs @@ -1,24 +1,13 @@ #![allow(missing_docs)] //! Crate-private implementation of PyClassObject -use std::cell::UnsafeCell; use std::marker::PhantomData; -use std::mem::ManuallyDrop; -use std::ptr::addr_of_mut; use std::sync::atomic::{AtomicUsize, Ordering}; -use crate::impl_::pyclass::{ - PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, -}; -use crate::internal::get_slot::TP_FREE; -use crate::type_object::PyNativeType; -use crate::types::PyType; -use crate::{ffi, PyTypeInfo, Python}; +use crate::ffi; +use crate::impl_::pyclass::PyClassImpl; -#[cfg(not(Py_LIMITED_API))] -use crate::types::PyTypeMethods; - -use super::{ptr_from_ref, PyBorrowError, PyBorrowMutError}; +use super::{PyBorrowError, PyBorrowMutError, PyObjectLayout}; pub trait PyClassMutability { // The storage for this inheritance layer. Only the first mutable class in @@ -209,385 +198,6 @@ where } } -#[repr(C)] -pub(crate) struct PyClassObjectContents { - pub(crate) value: ManuallyDrop>, - pub(crate) borrow_checker: ::Storage, - pub(crate) thread_checker: T::ThreadChecker, - pub(crate) dict: T::Dict, - pub(crate) weakref: T::WeakRef, -} - -impl PyClassObjectContents { - pub(crate) fn new(init: T) -> Self { - PyClassObjectContents { - value: ManuallyDrop::new(UnsafeCell::new(init)), - borrow_checker: ::Storage::new(), - thread_checker: T::ThreadChecker::new(), - dict: T::Dict::INIT, - weakref: T::WeakRef::INIT, - } - } - - unsafe fn dealloc(&mut self, py: Python<'_>, py_object: *mut ffi::PyObject) { - if self.thread_checker.can_drop(py) { - ManuallyDrop::drop(&mut self.value); - } - self.dict.clear_dict(py); - self.weakref.clear_weakrefs(py_object, py); - } -} - -/// Py_ssize_t may not be equal to isize on all platforms -fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { - #[allow(clippy::useless_conversion)] - value.try_into().expect("value should fit in Py_ssize_t") -} - -/// Functions for working with `PyObjects` recursively by re-interpreting the object -/// as being an instance of the most derived class through each base class until -/// the `BaseNativeType` is reached. -#[doc(hidden)] -pub trait PyObjectRecursiveOperations { - unsafe fn ensure_threadsafe(obj: &ffi::PyObject); - unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError>; - /// Cleanup then free the memory for `obj`. - /// - /// # Safety - /// - slf must be a valid pointer to an instance of a T or a subclass. - /// - slf must not be used after this call (as it will be freed). - unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject); -} - -/// Used to fill out `PyClassBaseType::RecursiveOperations` for instances of `PyClass` -pub struct PyClassRecursiveOperations(PhantomData); - -impl PyObjectRecursiveOperations for PyClassRecursiveOperations { - unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { - let contents = PyObjectLayout::get_contents::(obj); - contents.thread_checker.ensure(); - ::RecursiveOperations::ensure_threadsafe(obj); - } - - unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError> { - let contents = PyObjectLayout::get_contents::(obj); - if !contents.thread_checker.check() { - return Err(PyBorrowError { _private: () }); - } - ::RecursiveOperations::check_threadsafe(obj) - } - - unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { - // Safety: Python only calls tp_dealloc when no references to the object remain. - let contents = &mut *PyObjectLayout::get_contents_ptr::(obj); - contents.dealloc(py, obj); - ::RecursiveOperations::deallocate(py, obj); - } -} - -/// Used to fill out `PyClassBaseType::RecursiveOperations` for native types -pub struct PyNativeTypeRecursiveOperations(PhantomData); - -impl PyObjectRecursiveOperations - for PyNativeTypeRecursiveOperations -{ - unsafe fn ensure_threadsafe(_obj: &ffi::PyObject) {} - - unsafe fn check_threadsafe(_obj: &ffi::PyObject) -> Result<(), PyBorrowError> { - Ok(()) - } - - /// Call the destructor (`tp_dealloc`) of an object which is an instance of a - /// subclass of the native type `T`. - /// - /// Does not clear up any data from subtypes of `type_ptr` so it is assumed that those - /// destructors have been called first. - /// - /// [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - /// - /// # Safety - /// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. - /// - obj must not be used after this call (as it will be freed). - unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { - // the `BaseNativeType` of the object - let type_ptr = ::type_object_raw(py); - - // FIXME: there is potentially subtle issues here if the base is overwritten at runtime? To be investigated. - - // the 'most derived class' of `obj`. i.e. the result of calling `type(obj)`. - let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); - - // TODO(matt): is this correct? - // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free - let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); - let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); - if is_base_object || is_metaclass { - let tp_free = actual_type - .get_slot(TP_FREE) - .expect("base type should have tp_free"); - return tp_free(obj.cast()); - } - - // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. - #[cfg(not(Py_LIMITED_API))] - { - // FIXME: should this be using actual_type.tp_dealloc? - if let Some(dealloc) = (*type_ptr).tp_dealloc { - // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which - // assumes the exception is currently GC tracked, so we have to re-track - // before calling the dealloc so that it can safely call Py_GC_UNTRACK. - #[cfg(not(any(Py_3_11, PyPy)))] - if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { - ffi::PyObject_GC_Track(obj.cast()); - } - dealloc(obj); - } else { - (*actual_type.as_type_ptr()) - .tp_free - .expect("type missing tp_free")(obj.cast()); - } - } - - #[cfg(Py_LIMITED_API)] - unreachable!("subclassing native types is not possible with the `abi3` feature"); - } -} - -/// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). -#[doc(hidden)] -pub(crate) mod opaque_layout { - use super::PyClassObjectContents; - use crate::ffi; - use crate::impl_::pyclass::PyClassImpl; - - #[cfg(Py_3_12)] - pub fn get_contents_ptr( - obj: *mut ffi::PyObject, - ) -> *mut PyClassObjectContents { - #[cfg(Py_3_12)] - { - // TODO(matt): this needs to be ::type_object_raw(py) - let type_obj = unsafe { ffi::Py_TYPE(obj) }; - assert!(!type_obj.is_null()); - let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; - assert!(!pointer.is_null()); - pointer as *mut PyClassObjectContents - } - - #[cfg(not(Py_3_12))] - panic_unsupported(); - } - - #[inline(always)] - #[cfg(not(Py_3_12))] - fn panic_unsupported() { - panic!("opaque layout not supported until python 3.12"); - } -} - -/// Utilities for working with `PyObject` objects that utilise the standard layout for python extensions, -/// where the base class is placed at the beginning of a `repr(C)` struct. -#[doc(hidden)] -pub(crate) mod static_layout { - use crate::{ - impl_::pyclass::{PyClassBaseType, PyClassImpl}, - type_object::{PyLayout, PySizedLayout}, - }; - - use super::PyClassObjectContents; - - // The layout of a `PyObject` that uses the static layout - #[repr(C)] - pub struct PyStaticClassLayout { - pub(crate) ob_base: ::StaticLayout, - pub(crate) contents: PyClassObjectContents, - } - - unsafe impl PyLayout for PyStaticClassLayout {} - - /// Base layout of PyClassObject with a known sized base type. - /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. - #[doc(hidden)] - #[repr(C)] - pub struct PyStaticNativeLayout { - ob_base: T, - } - - unsafe impl PyLayout for PyStaticNativeLayout where U: PySizedLayout {} - - /// a struct for use with opaque native types to indicate that they - /// cannot be used as part of a static layout. - #[repr(C)] - pub struct InvalidStaticLayout; - - /// This is valid insofar as casting a `*mut ffi::PyObject` to `*mut InvalidStaticLayout` is valid - /// since nothing can actually be read by dereferencing. - unsafe impl PyLayout for InvalidStaticLayout {} -} - -/// Functions for working with `PyObject`s -pub(crate) struct PyObjectLayout {} - -impl PyObjectLayout { - /// Obtain a pointer to the contents of a `PyObject` of type `T`. - /// - /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_contents_ptr( - obj: *mut ffi::PyObject, - ) -> *mut PyClassObjectContents { - debug_assert!(!obj.is_null()); - if ::OPAQUE { - opaque_layout::get_contents_ptr(obj) - } else { - let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); - // indicates `ob_base` has type InvalidBaseLayout - debug_assert_ne!( - std::mem::offset_of!(static_layout::PyStaticClassLayout, contents), - 0, - "invalid ob_base found" - ); - addr_of_mut!((*obj).contents) - } - } - - pub(crate) unsafe fn get_contents( - obj: &ffi::PyObject, - ) -> &PyClassObjectContents { - &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut()).cast_const() - } - - /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. - /// - /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_data_ptr(obj: *mut ffi::PyObject) -> *mut T { - let contents = PyObjectLayout::get_contents_ptr::(obj); - (*contents).value.get() - } - - pub(crate) unsafe fn get_data(obj: &ffi::PyObject) -> &T { - &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut()) - } - - pub(crate) unsafe fn get_borrow_checker( - obj: &ffi::PyObject, - ) -> &::Checker { - T::PyClassMutability::borrow_checker(obj) - } - - pub(crate) unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { - PyClassRecursiveOperations::::ensure_threadsafe(obj) - } - - pub(crate) unsafe fn check_threadsafe( - obj: &ffi::PyObject, - ) -> Result<(), PyBorrowError> { - PyClassRecursiveOperations::::check_threadsafe(obj) - } - - /// Clean up then free the memory associated with `obj`. - /// - /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { - unsafe { - PyClassRecursiveOperations::::deallocate(py, obj); - }; - } - - /// Clean up then free the memory associated with `obj`. - /// - /// Use instead of `deallocate()` if `T` has the `Py_TPFLAGS_HAVE_GC` flag set. - /// - /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - pub(crate) fn deallocate_with_gc(py: Python<'_>, obj: *mut ffi::PyObject) { - unsafe { - // TODO(matt): verify T has flag set - #[cfg(not(PyPy))] - { - ffi::PyObject_GC_UnTrack(obj.cast()); - } - PyClassRecursiveOperations::::deallocate(py, obj); - }; - } - - /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` - /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) - pub(crate) fn basicsize() -> ffi::Py_ssize_t { - if ::OPAQUE { - #[cfg(Py_3_12)] - { - // negative to indicate 'extra' space that python will allocate - // specifically for `T` excluding the base class - -usize_to_py_ssize(std::mem::size_of::>()) - } - - #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); - } else { - usize_to_py_ssize(std::mem::size_of::>()) - } - } - - /// Gets the offset of the contents from the start of the struct in bytes. - pub(crate) fn contents_offset() -> PyObjectOffset { - if ::OPAQUE { - #[cfg(Py_3_12)] - { - PyObjectOffset::Relative(0) - } - - #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); - } else { - PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( - static_layout::PyStaticClassLayout, - contents - ))) - } - } - - /// Gets the offset of the dictionary from the start of the struct in bytes. - pub(crate) fn dict_offset() -> PyObjectOffset { - if ::OPAQUE { - #[cfg(Py_3_12)] - { - PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( - PyClassObjectContents, - dict - ))) - } - - #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); - } else { - let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) - + memoffset::offset_of!(PyClassObjectContents, dict); - - PyObjectOffset::Absolute(usize_to_py_ssize(offset)) - } - } - - /// Gets the offset of the weakref list from the start of the struct in bytes. - pub(crate) fn weaklist_offset() -> PyObjectOffset { - if ::OPAQUE { - #[cfg(Py_3_12)] - { - PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( - PyClassObjectContents, - weakref - ))) - } - - #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); - } else { - let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) - + memoffset::offset_of!(PyClassObjectContents, weakref); - - PyObjectOffset::Absolute(usize_to_py_ssize(offset)) - } - } -} - #[cfg(test)] #[cfg(feature = "macros")] mod tests { @@ -647,27 +257,6 @@ mod tests { #[pyclass(crate = "crate", extends = BaseWithData)] struct ChildWithoutData; - #[test] - fn test_inherited_size() { - let base_size = PyObjectLayout::basicsize::(); - assert!(base_size > 0); // negative indicates variable sized - assert_eq!(base_size, PyObjectLayout::basicsize::()); - assert!(base_size < PyObjectLayout::basicsize::()); - } - - #[test] - fn test_invalid_base() { - assert_eq!(std::mem::size_of::(), 0); - - #[repr(C)] - struct InvalidLayout { - ob_base: static_layout::InvalidStaticLayout, - contents: u8, - } - - assert_eq!(std::mem::offset_of!(InvalidLayout, contents), 0); - } - fn assert_mutable>() {} fn assert_immutable>() {} fn assert_mutable_with_mutable_ancestor< @@ -852,6 +441,8 @@ mod tests { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_thread_safety_2() { + use std::cell::UnsafeCell; + struct SyncUnsafeCell(UnsafeCell); unsafe impl Sync for SyncUnsafeCell {} diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs new file mode 100644 index 00000000000..485976715ce --- /dev/null +++ b/src/pycell/layout.rs @@ -0,0 +1,451 @@ +#![allow(missing_docs)] +//! Crate-private implementation of how PyClassObjects are laid out in memory and how to access data from raw PyObjects + +use std::cell::UnsafeCell; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::ptr::addr_of_mut; + +use crate::impl_::pyclass::{ + PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, + PyClassWeakRef, PyObjectOffset, +}; +use crate::pycell::borrow_checker::{PyClassBorrowChecker, GetBorrowChecker}; +use crate::internal::get_slot::TP_FREE; +use crate::type_object::PyNativeType; +use crate::types::PyType; +use crate::{ffi, PyTypeInfo, Python}; + +#[cfg(not(Py_LIMITED_API))] +use crate::types::PyTypeMethods; + +use super::borrow_checker::PyClassMutability; +use super::{ptr_from_ref, PyBorrowError}; + +/// The data of a `ffi::PyObject` specifically relating to type `T`. +/// +/// In an inheritance hierarchy where `#[pyclass(extends=PyDict)] struct A;` and `#[pyclass(extends=A)] struct B;` +/// a `ffi::PyObject` of type `B` has separate memory for `ffi::PyDictObject` (the base native type) and +/// `PyClassObjectContents` and `PyClassObjectContents`. The memory associated with `A` or `B` can be obtained +/// using `PyObjectLayout::get_contents::()` (where `T=A` or `T=B`). +#[repr(C)] +pub(crate) struct PyClassObjectContents { + /// The data associated with the user-defined struct annotated with `#[pyclass]` + pub(crate) value: ManuallyDrop>, + pub(crate) borrow_checker: ::Storage, + pub(crate) thread_checker: T::ThreadChecker, + pub(crate) dict: T::Dict, + pub(crate) weakref: T::WeakRef, +} + +impl PyClassObjectContents { + pub(crate) fn new(init: T) -> Self { + PyClassObjectContents { + value: ManuallyDrop::new(UnsafeCell::new(init)), + borrow_checker: ::Storage::new(), + thread_checker: T::ThreadChecker::new(), + dict: T::Dict::INIT, + weakref: T::WeakRef::INIT, + } + } + + unsafe fn dealloc(&mut self, py: Python<'_>, py_object: *mut ffi::PyObject) { + if self.thread_checker.can_drop(py) { + ManuallyDrop::drop(&mut self.value); + } + self.dict.clear_dict(py); + self.weakref.clear_weakrefs(py_object, py); + } +} + +/// Functions for working with `PyObjects` recursively by re-interpreting the object +/// as being an instance of the most derived class through each base class until +/// the `BaseNativeType` is reached. +/// +/// E.g. if `#[pyclass(extends=PyDict)] struct A;` and `#[pyclass(extends=A)] struct B;` +/// then calling a method on a PyObject of type `B` will call the method for `B`, then `A`, then `PyDict`. +#[doc(hidden)] +pub trait PyObjectRecursiveOperations { + unsafe fn ensure_threadsafe(obj: &ffi::PyObject); + unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError>; + /// Cleanup then free the memory for `obj`. + /// + /// # Safety + /// - slf must be a valid pointer to an instance of a T or a subclass. + /// - slf must not be used after this call (as it will be freed). + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject); +} + +/// Used to fill out `PyClassBaseType::RecursiveOperations` for instances of `PyClass` +pub struct PyClassRecursiveOperations(PhantomData); + +impl PyObjectRecursiveOperations for PyClassRecursiveOperations { + unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { + let contents = PyObjectLayout::get_contents::(obj); + contents.thread_checker.ensure(); + ::RecursiveOperations::ensure_threadsafe(obj); + } + + unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError> { + let contents = PyObjectLayout::get_contents::(obj); + if !contents.thread_checker.check() { + return Err(PyBorrowError { _private: () }); + } + ::RecursiveOperations::check_threadsafe(obj) + } + + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + // Safety: Python only calls tp_dealloc when no references to the object remain. + let contents = &mut *PyObjectLayout::get_contents_ptr::(obj); + contents.dealloc(py, obj); + ::RecursiveOperations::deallocate(py, obj); + } +} + +/// Used to fill out `PyClassBaseType::RecursiveOperations` for native types +pub struct PyNativeTypeRecursiveOperations(PhantomData); + +impl PyObjectRecursiveOperations + for PyNativeTypeRecursiveOperations +{ + unsafe fn ensure_threadsafe(_obj: &ffi::PyObject) {} + + unsafe fn check_threadsafe(_obj: &ffi::PyObject) -> Result<(), PyBorrowError> { + Ok(()) + } + + /// Call the destructor (`tp_dealloc`) of an object which is an instance of a + /// subclass of the native type `T`. + /// + /// Does not clear up any data from subtypes of `type_ptr` so it is assumed that those + /// destructors have been called first. + /// + /// [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + /// + /// # Safety + /// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. + /// - obj must not be used after this call (as it will be freed). + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + // the `BaseNativeType` of the object + let type_ptr = ::type_object_raw(py); + + // FIXME: there is potentially subtle issues here if the base is overwritten at runtime? To be investigated. + + // the 'most derived class' of `obj`. i.e. the result of calling `type(obj)`. + let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); + + // TODO(matt): is this correct? + // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free + let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); + let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); + if is_base_object || is_metaclass { + let tp_free = actual_type + .get_slot(TP_FREE) + .expect("base type should have tp_free"); + return tp_free(obj.cast()); + } + + // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. + #[cfg(not(Py_LIMITED_API))] + { + // FIXME: should this be using actual_type.tp_dealloc? + if let Some(dealloc) = (*type_ptr).tp_dealloc { + // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which + // assumes the exception is currently GC tracked, so we have to re-track + // before calling the dealloc so that it can safely call Py_GC_UNTRACK. + #[cfg(not(any(Py_3_11, PyPy)))] + if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { + ffi::PyObject_GC_Track(obj.cast()); + } + dealloc(obj); + } else { + (*actual_type.as_type_ptr()) + .tp_free + .expect("type missing tp_free")(obj.cast()); + } + } + + #[cfg(Py_LIMITED_API)] + unreachable!("subclassing native types is not possible with the `abi3` feature"); + } +} + +/// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). +#[doc(hidden)] +pub(crate) mod opaque_layout { + use super::PyClassObjectContents; + use crate::ffi; + use crate::impl_::pyclass::PyClassImpl; + + #[cfg(Py_3_12)] + pub fn get_contents_ptr( + obj: *mut ffi::PyObject, + ) -> *mut PyClassObjectContents { + #[cfg(Py_3_12)] + { + // TODO(matt): this needs to be ::type_object_raw(py) + let type_obj = unsafe { ffi::Py_TYPE(obj) }; + assert!(!type_obj.is_null()); + let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; + assert!(!pointer.is_null()); + pointer as *mut PyClassObjectContents + } + + #[cfg(not(Py_3_12))] + panic_unsupported(); + } + + #[inline(always)] + #[cfg(not(Py_3_12))] + fn panic_unsupported() { + panic!("opaque layout not supported until python 3.12"); + } +} + +/// Utilities for working with `PyObject` objects that utilise the standard layout for python extensions, +/// where the base class is placed at the beginning of a `repr(C)` struct. +#[doc(hidden)] +pub(crate) mod static_layout { + use crate::{ + impl_::pyclass::{PyClassBaseType, PyClassImpl}, + type_object::{PyLayout, PySizedLayout}, + }; + + use super::PyClassObjectContents; + + // The layout of a `PyObject` that uses the static layout + #[repr(C)] + pub struct PyStaticClassLayout { + pub(crate) ob_base: ::StaticLayout, + pub(crate) contents: PyClassObjectContents, + } + + unsafe impl PyLayout for PyStaticClassLayout {} + + /// Base layout of PyClassObject with a known sized base type. + /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. + #[doc(hidden)] + #[repr(C)] + pub struct PyStaticNativeLayout { + ob_base: T, + } + + unsafe impl PyLayout for PyStaticNativeLayout where U: PySizedLayout {} + + /// a struct for use with opaque native types to indicate that they + /// cannot be used as part of a static layout. + #[repr(C)] + pub struct InvalidStaticLayout; + + /// This is valid insofar as casting a `*mut ffi::PyObject` to `*mut InvalidStaticLayout` is valid + /// since nothing can actually be read by dereferencing. + unsafe impl PyLayout for InvalidStaticLayout {} +} + +/// Functions for working with `PyObject`s +pub(crate) struct PyObjectLayout {} + +impl PyObjectLayout { + /// Obtain a pointer to the contents of a `PyObject` of type `T`. + /// + /// Safety: the provided object must be valid and have the layout indicated by `T` + pub(crate) unsafe fn get_contents_ptr( + obj: *mut ffi::PyObject, + ) -> *mut PyClassObjectContents { + debug_assert!(!obj.is_null()); + if ::OPAQUE { + opaque_layout::get_contents_ptr(obj) + } else { + let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); + // indicates `ob_base` has type InvalidBaseLayout + debug_assert_ne!( + std::mem::offset_of!(static_layout::PyStaticClassLayout, contents), + 0, + "invalid ob_base found" + ); + addr_of_mut!((*obj).contents) + } + } + + pub(crate) unsafe fn get_contents( + obj: &ffi::PyObject, + ) -> &PyClassObjectContents { + &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut()).cast_const() + } + + /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. + /// + /// Safety: the provided object must be valid and have the layout indicated by `T` + pub(crate) unsafe fn get_data_ptr(obj: *mut ffi::PyObject) -> *mut T { + let contents = PyObjectLayout::get_contents_ptr::(obj); + (*contents).value.get() + } + + pub(crate) unsafe fn get_data(obj: &ffi::PyObject) -> &T { + &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut()) + } + + pub(crate) unsafe fn get_borrow_checker( + obj: &ffi::PyObject, + ) -> &::Checker { + T::PyClassMutability::borrow_checker(obj) + } + + pub(crate) unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { + PyClassRecursiveOperations::::ensure_threadsafe(obj) + } + + pub(crate) unsafe fn check_threadsafe( + obj: &ffi::PyObject, + ) -> Result<(), PyBorrowError> { + PyClassRecursiveOperations::::check_threadsafe(obj) + } + + /// Clean up then free the memory associated with `obj`. + /// + /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + unsafe { + PyClassRecursiveOperations::::deallocate(py, obj); + }; + } + + /// Clean up then free the memory associated with `obj`. + /// + /// Use instead of `deallocate()` if `T` has the `Py_TPFLAGS_HAVE_GC` flag set. + /// + /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + pub(crate) fn deallocate_with_gc(py: Python<'_>, obj: *mut ffi::PyObject) { + unsafe { + // TODO(matt): verify T has flag set + #[cfg(not(PyPy))] + { + ffi::PyObject_GC_UnTrack(obj.cast()); + } + PyClassRecursiveOperations::::deallocate(py, obj); + }; + } + + /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` + /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) + pub(crate) fn basicsize() -> ffi::Py_ssize_t { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + // negative to indicate 'extra' space that python will allocate + // specifically for `T` excluding the base class + -usize_to_py_ssize(std::mem::size_of::>()) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + usize_to_py_ssize(std::mem::size_of::>()) + } + } + + /// Gets the offset of the contents from the start of the struct in bytes. + pub(crate) fn contents_offset() -> PyObjectOffset { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(0) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( + static_layout::PyStaticClassLayout, + contents + ))) + } + } + + /// Gets the offset of the dictionary from the start of the struct in bytes. + pub(crate) fn dict_offset() -> PyObjectOffset { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + dict + ))) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + + memoffset::offset_of!(PyClassObjectContents, dict); + + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) + } + } + + /// Gets the offset of the weakref list from the start of the struct in bytes. + pub(crate) fn weaklist_offset() -> PyObjectOffset { + if ::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + weakref + ))) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported(); + } else { + let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + + memoffset::offset_of!(PyClassObjectContents, weakref); + + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) + } + } +} + +/// Py_ssize_t may not be equal to isize on all platforms +fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { + #[allow(clippy::useless_conversion)] + value.try_into().expect("value should fit in Py_ssize_t") +} + + +#[cfg(test)] +#[cfg(feature = "macros")] +mod tests { + use super::*; + + use crate::prelude::*; + + #[pyclass(crate = "crate", subclass)] + struct BaseWithData(#[allow(unused)] u64); + + #[pyclass(crate = "crate", extends = BaseWithData)] + struct ChildWithData(#[allow(unused)] u64); + + #[pyclass(crate = "crate", extends = BaseWithData)] + struct ChildWithoutData; + + #[test] + fn test_inherited_size() { + let base_size = PyObjectLayout::basicsize::(); + assert!(base_size > 0); // negative indicates variable sized + assert_eq!(base_size, PyObjectLayout::basicsize::()); + assert!(base_size < PyObjectLayout::basicsize::()); + } + + #[test] + fn test_invalid_base() { + assert_eq!(std::mem::size_of::(), 0); + + #[repr(C)] + struct InvalidLayout { + ob_base: static_layout::InvalidStaticLayout, + contents: u8, + } + + assert_eq!(std::mem::offset_of!(InvalidLayout, contents), 0); + } +} diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 5c2169ceb07..9ae504bbe09 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -10,7 +10,7 @@ use crate::{ trampoline::trampoline, }, internal_tricks::ptr_from_ref, - pycell::impl_::PyObjectLayout, + pycell::layout::PyObjectLayout, types::{typeobject::PyTypeMethods, PyType}, Py, PyClass, PyResult, PyTypeInfo, Python, }; diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 3187dac3e60..17928ff9995 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -3,10 +3,10 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::PyClassBaseType; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; -use crate::pycell::impl_::PyObjectLayout; +use crate::pycell::layout::PyObjectLayout; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; -use crate::{ffi::PyTypeObject, pycell::impl_::PyClassObjectContents}; +use crate::{ffi::PyTypeObject, pycell::layout::PyClassObjectContents}; use std::marker::PhantomData; /// Initializer for our `#[pyclass]` system. diff --git a/src/types/any.rs b/src/types/any.rs index 492929fe4cd..23f271dc2a8 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -52,7 +52,7 @@ impl crate::impl_::pyclass::PyClassBaseType for PyAny { type BaseNativeType = PyAny; type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; - type PyClassMutability = crate::pycell::impl_::ImmutableClass; + type PyClassMutability = crate::pycell::borrow_checker::ImmutableClass; } /// This trait represents the Python APIs which are usable on all Python objects. diff --git a/src/types/mod.rs b/src/types/mod.rs index 58d8732ac8f..677f25d803f 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -217,7 +217,7 @@ macro_rules! pyobject_subclassable_native_type { type BaseNativeType = $name; type RecursiveOperations = $crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = $crate::impl_::pyclass_init::PyNativeTypeInitializer; - type PyClassMutability = $crate::pycell::impl_::ImmutableClass; + type PyClassMutability = $crate::pycell::borrow_checker::ImmutableClass; } } } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 4744a003339..915f37c655b 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -30,7 +30,7 @@ impl crate::impl_::pyclass::PyClassBaseType for PyType { type BaseNativeType = PyType; type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; - type PyClassMutability = crate::pycell::impl_::ImmutableClass; + type PyClassMutability = crate::pycell::borrow_checker::ImmutableClass; } impl PyType { From e8e1a1eba694cca02c4243c17ac77acdd764720c Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 10 Nov 2024 15:25:05 +0000 Subject: [PATCH 30/41] misc improvements --- pyo3-macros-backend/src/pymethod.rs | 2 +- src/impl_/trampoline.rs | 7 ++++++- src/pycell.rs | 4 ++-- src/pycell/layout.rs | 6 ++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 054d0d68e2d..5d618d4729b 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -421,11 +421,11 @@ fn impl_init_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result ::std::os::raw::c_int { - #pyo3_path::impl_::pyclass_init::initialize_with_default::<#cls>(slf); #pyo3_path::impl_::trampoline::initproc( slf, args, kwargs, + #pyo3_path::impl_::pyclass_init::initialize_with_default::<#cls>, #cls::#wrapper_ident ) } diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index f29e6dce395..52726cfaf03 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -129,6 +129,8 @@ pub unsafe fn initproc( slf: *mut ffi::PyObject, args: *mut ffi::PyObject, kwargs: *mut ffi::PyObject, + // initializes the object to a valid state before running the user-defined init function + initialize: unsafe fn(*mut ffi::PyObject), f: for<'py> unsafe fn( Python<'py>, *mut ffi::PyObject, @@ -137,7 +139,10 @@ pub unsafe fn initproc( ) -> PyResult<*mut ffi::PyObject>, ) -> c_int { // the map() discards the success value of `f` and converts to the success return value for tp_init (0) - trampoline(|py| f(py, slf, args, kwargs).map(|_| 0)) + trampoline(|py| { + initialize(slf); + f(py, slf, args, kwargs).map(|_| 0) + }) } #[cfg(any(not(Py_LIMITED_API), Py_3_11))] diff --git a/src/pycell.rs b/src/pycell.rs index c046592d279..87204058a14 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -574,9 +574,9 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { .map(|_| Self { inner: obj.clone() }) } - pub(crate) fn downgrade(slf: &Self) -> &PyRef<'py, T> { + pub(crate) fn downgrade(&self) -> &PyRef<'py, T> { // `PyRefMut` and `PyRef` have the same layout - unsafe { &*ptr_from_ref(slf).cast() } + unsafe { &*ptr_from_ref(self).cast() } } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 485976715ce..03727f234c8 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -34,7 +34,9 @@ pub(crate) struct PyClassObjectContents { pub(crate) value: ManuallyDrop>, pub(crate) borrow_checker: ::Storage, pub(crate) thread_checker: T::ThreadChecker, + /// A pointer to a `PyObject` if `T` is annotated with `#[pyclass(dict)]` and a zero-sized field otherwise. pub(crate) dict: T::Dict, + /// A pointer to a `PyObject` if `T` is annotated with `#[pyclass(weakref)]` and a zero-sized field otherwise. pub(crate) weakref: T::WeakRef, } @@ -187,8 +189,8 @@ pub(crate) mod opaque_layout { let type_obj = unsafe { ffi::Py_TYPE(obj) }; assert!(!type_obj.is_null()); let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; - assert!(!pointer.is_null()); - pointer as *mut PyClassObjectContents + assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); + pointer.cast() } #[cfg(not(Py_3_12))] From e72dded45e0f16e305d9f96735e0f2bf565f5ab7 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 10 Nov 2024 22:00:21 +0000 Subject: [PATCH 31/41] add method to PyTypeInfo to obtain the type object without the GIL --- pyo3-macros-backend/src/pyclass.rs | 6 ++ src/exceptions.rs | 67 +++++--------- src/impl_/pyclass/lazy_type_object.rs | 14 +++ src/sync.rs | 11 +++ src/type_object.rs | 14 ++- src/types/any.rs | 2 +- src/types/boolobject.rs | 3 +- src/types/bytearray.rs | 3 +- src/types/bytes.rs | 3 +- src/types/capsule.rs | 3 +- src/types/code.rs | 7 +- src/types/complex.rs | 9 +- src/types/datetime.rs | 37 +++++++- src/types/dict.rs | 32 +++---- src/types/ellipsis.rs | 4 + src/types/float.rs | 9 +- src/types/frame.rs | 7 +- src/types/frozenset.rs | 16 +--- src/types/function.rs | 7 +- src/types/list.rs | 3 +- src/types/mappingproxy.rs | 7 +- src/types/memoryview.rs | 3 +- src/types/mod.rs | 127 +++++++++++++++++++++----- src/types/module.rs | 3 +- src/types/none.rs | 4 + src/types/notimplemented.rs | 4 + src/types/num.rs | 3 +- src/types/pysuper.rs | 6 +- src/types/set.rs | 17 +--- src/types/slice.rs | 8 +- src/types/string.rs | 3 +- src/types/traceback.rs | 7 +- src/types/tuple.rs | 3 +- src/types/typeobject.rs | 6 +- src/types/weakref/reference.rs | 3 +- 35 files changed, 281 insertions(+), 180 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index d5a71a91cbb..98ca9b653b9 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1820,6 +1820,12 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre .get_or_init(py) .as_type_ptr() } + + #[inline] + fn try_get_type_object_raw() -> ::std::option::Option<*mut #pyo3_path::ffi::PyTypeObject> { + <#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object() + .try_get_raw() + } } } } diff --git a/src/exceptions.rs b/src/exceptions.rs index b682d7a0563..4f358ced1f7 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -87,18 +87,13 @@ macro_rules! import_exception { $crate::pyobject_native_type_core!( $name, - $name::type_object_raw, #module=::std::option::Option::Some(stringify!($module)) ); - - impl $name { - fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::types::PyTypeMethods; - static TYPE_OBJECT: $crate::impl_::exceptions::ImportedExceptionTypeObject = - $crate::impl_::exceptions::ImportedExceptionTypeObject::new(stringify!($module), stringify!($name)); - TYPE_OBJECT.get(py).as_type_ptr() - } - } + $crate::pyobject_native_type_object_methods!( + $name, + #import_module=$module, + #import_name=$name + ); }; } @@ -123,24 +118,16 @@ macro_rules! import_exception_bound { $crate::pyobject_native_type_info!( $name, - $name::type_object_raw, ::std::option::Option::Some(stringify!($module)), false ); + $crate::pyobject_native_type_object_methods!( + $name, + #import_module=$module, + #import_name=$name + ); impl $crate::types::DerefToPyAny for $name {} - - impl $name { - fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::types::PyTypeMethods; - static TYPE_OBJECT: $crate::impl_::exceptions::ImportedExceptionTypeObject = - $crate::impl_::exceptions::ImportedExceptionTypeObject::new( - stringify!($module), - stringify!($name), - ); - TYPE_OBJECT.get(py).as_type_ptr() - } - } }; } @@ -247,28 +234,20 @@ macro_rules! create_exception_type_object { ($module: expr, $name: ident, $base: ty, $doc: expr) => { $crate::pyobject_native_type_core!( $name, - $name::type_object_raw, #module=::std::option::Option::Some(stringify!($module)) ); - - impl $name { - fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::sync::GILOnceCell; - static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = - GILOnceCell::new(); - - TYPE_OBJECT - .get_or_init(py, || - $crate::PyErr::new_type( - py, - $crate::ffi::c_str!(concat!(stringify!($module), ".", stringify!($name))), - $doc, - ::std::option::Option::Some(&py.get_type::<$base>()), - ::std::option::Option::None, - ).expect("Failed to initialize new exception type.") - ).as_ptr() as *mut $crate::ffi::PyTypeObject + $crate::pyobject_native_type_object_methods!( + $name, + #create=|py| { + $crate::PyErr::new_type( + py, + $crate::ffi::c_str!(concat!(stringify!($module), ".", stringify!($name))), + $doc, + ::std::option::Option::Some(&py.get_type::<$base>()), + ::std::option::Option::None, + ).expect("Failed to initialize new exception type.") } - } + ); }; } @@ -279,7 +258,8 @@ macro_rules! impl_native_exception ( pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, |_py| unsafe { $crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject } $(, #checkfunction=$checkfunction)?); + $crate::pyobject_native_type!($name, $layout $(, #checkfunction=$checkfunction)?); + $crate::pyobject_native_type_object_methods!($name, #global_ptr=$crate::ffi::$exc_name); $crate::pyobject_subclassable_native_type!($name, $layout); ); ($name:ident, $exc_name:ident, $doc:expr) => ( @@ -378,6 +358,7 @@ impl_native_exception!( ffi::PyBaseExceptionObject, #checkfunction=ffi::PyExceptionInstance_Check ); + impl_native_exception!(PyException, PyExc_Exception, native_doc!("Exception")); impl_native_exception!( PyStopAsyncIteration, diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index d3bede7b2f3..6b7832bbf74 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -4,6 +4,8 @@ use std::{ thread::{self, ThreadId}, }; +use pyo3_ffi::PyTypeObject; + use crate::{ exceptions::PyRuntimeError, ffi, @@ -61,6 +63,10 @@ impl LazyTypeObject { self.0 .get_or_try_init(py, create_type_object::, T::NAME, T::items_iter()) } + + pub fn try_get_raw(&self) -> Option<*mut PyTypeObject> { + self.0.try_get_raw() + } } impl LazyTypeObjectInner { @@ -92,6 +98,14 @@ impl LazyTypeObjectInner { }) } + pub fn try_get_raw(&self) -> Option<*mut ffi::PyTypeObject> { + unsafe { + self.value + .get_raw() + .map(|obj| (*obj).type_object.as_ptr().cast::()) + } + } + fn ensure_init( &self, type_object: &Bound<'_, PyType>, diff --git a/src/sync.rs b/src/sync.rs index 0845eaf8cec..5b9ce826d98 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -179,6 +179,17 @@ impl GILOnceCell { } } + /// Get a pointer to the contained value, or `None` if the cell has not yet been written. + #[inline] + pub fn get_raw(&self) -> Option<*const T> { + if self.once.is_completed() { + // SAFETY: the cell has been written. + Some(unsafe { (*self.data.get()).as_ptr() }) + } else { + None + } + } + /// Get a reference to the contained value, initializing it if needed using the provided /// closure. /// diff --git a/src/type_object.rs b/src/type_object.rs index 46b8c32e91e..2f952699382 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -42,8 +42,8 @@ pub trait PySizedLayout: PyLayout + Sized {} /// /// # Safety /// -/// Implementations must provide an implementation for `type_object_raw` which infallibly produces a -/// non-null pointer to the corresponding Python type object. +/// Implementations must return the correct non-null `PyTypeObject` pointer corresponding to the type of `Self` +/// from `type_object_raw` and `try_get_type_object_raw`. pub unsafe trait PyTypeInfo: Sized { /// Class name. const NAME: &'static str; @@ -52,13 +52,21 @@ pub unsafe trait PyTypeInfo: Sized { const MODULE: Option<&'static str>; /// Whether classes that extend from this type must use the 'opaque type' extension mechanism - /// rather than using the standard mechanism of placing the data for this type at the beginning + /// rather than using the standard mechanism of placing the data for this type at the end /// of a new `repr(C)` struct const OPAQUE: bool; /// Returns the `PyTypeObject` instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; + /// Returns the `PyTypeObject` instance for this type if it is known statically or has already + /// been initialized (by calling `type_object_raw()`). + /// + /// # Safety + /// - It is valid to always return Some. + /// - It is not valid to return None once `type_object_raw()` has been called. + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject>; + /// Returns the safe abstraction over the type object. #[inline] fn type_object(py: Python<'_>) -> Bound<'_, PyType> { diff --git a/src/types/any.rs b/src/types/any.rs index 23f271dc2a8..1d16e80d7ab 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -39,11 +39,11 @@ fn PyObject_Check(_: *mut ffi::PyObject) -> c_int { pyobject_native_type_info!( PyAny, - pyobject_native_static_type_object!(ffi::PyBaseObject_Type), Some("builtins"), false, #checkfunction=PyObject_Check ); +pyobject_native_type_object_methods!(PyAny, #global=ffi::PyBaseObject_Type); pyobject_native_type_marker!(PyAny); pyobject_native_type_sized!(PyAny, ffi::PyObject); // We cannot use `pyobject_subclassable_native_type!()` because it cfgs out on `Py_LIMITED_API`. diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 53043fa798c..c67d4a2e146 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -22,7 +22,8 @@ use std::convert::Infallible; #[repr(transparent)] pub struct PyBool(PyAny); -pyobject_native_type!(PyBool, ffi::PyObject, pyobject_native_static_type_object!(ffi::PyBool_Type), #checkfunction=ffi::PyBool_Check); +pyobject_native_type!(PyBool, ffi::PyObject, #checkfunction=ffi::PyBool_Check); +pyobject_native_type_object_methods!(PyBool, #global=ffi::PyBool_Type); impl PyBool { /// Depending on `val`, returns `true` or `false`. diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index d1bbd0ac7e4..d7fe64b3b8a 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -16,7 +16,8 @@ use std::slice; #[repr(transparent)] pub struct PyByteArray(PyAny); -pyobject_native_type_core!(PyByteArray, pyobject_native_static_type_object!(ffi::PyByteArray_Type), #checkfunction=ffi::PyByteArray_Check); +pyobject_native_type_core!(PyByteArray, #checkfunction=ffi::PyByteArray_Check); +pyobject_native_type_object_methods!(PyByteArray, #global=ffi::PyByteArray_Type); impl PyByteArray { /// Creates a new Python bytearray object. diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 77b1d2b735d..195e7f25884 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -47,7 +47,8 @@ use std::str; #[repr(transparent)] pub struct PyBytes(PyAny); -pyobject_native_type_core!(PyBytes, pyobject_native_static_type_object!(ffi::PyBytes_Type), #checkfunction=ffi::PyBytes_Check); +pyobject_native_type_core!(PyBytes, #checkfunction=ffi::PyBytes_Check); +pyobject_native_type_object_methods!(PyBytes, #global=ffi::PyBytes_Type); impl PyBytes { /// Creates a new Python bytestring object. diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 9d9e6e4eb72..494597eafd7 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -47,7 +47,8 @@ use std::os::raw::{c_char, c_int, c_void}; #[repr(transparent)] pub struct PyCapsule(PyAny); -pyobject_native_type_core!(PyCapsule, pyobject_native_static_type_object!(ffi::PyCapsule_Type), #checkfunction=ffi::PyCapsule_CheckExact); +pyobject_native_type_core!(PyCapsule, #checkfunction=ffi::PyCapsule_CheckExact); +pyobject_native_type_object_methods!(PyCapsule, #global=ffi::PyCapsule_Type); impl PyCapsule { /// Constructs a new capsule whose contents are `value`, associated with `name`. diff --git a/src/types/code.rs b/src/types/code.rs index 0c1683c75be..412314ba571 100644 --- a/src/types/code.rs +++ b/src/types/code.rs @@ -8,11 +8,8 @@ use crate::PyAny; #[repr(transparent)] pub struct PyCode(PyAny); -pyobject_native_type_core!( - PyCode, - pyobject_native_static_type_object!(ffi::PyCode_Type), - #checkfunction=ffi::PyCode_Check -); +pyobject_native_type_core!(PyCode, #checkfunction=ffi::PyCode_Check); +pyobject_native_type_object_methods!(PyCode, #global=ffi::PyCode_Type); #[cfg(test)] mod tests { diff --git a/src/types/complex.rs b/src/types/complex.rs index 58651569b47..105ed67aeaf 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -20,13 +20,8 @@ use std::os::raw::c_double; pub struct PyComplex(PyAny); pyobject_subclassable_native_type!(PyComplex, ffi::PyComplexObject); - -pyobject_native_type!( - PyComplex, - ffi::PyComplexObject, - pyobject_native_static_type_object!(ffi::PyComplex_Type), - #checkfunction=ffi::PyComplex_Check -); +pyobject_native_type!(PyComplex, ffi::PyComplexObject, #checkfunction=ffi::PyComplex_Check); +pyobject_native_type_object_methods!(PyComplex, #global=ffi::PyComplex_Type); impl PyComplex { /// Creates a new `PyComplex` from the given real and imaginary values. diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 8ab512ac466..ad77bfae630 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -30,6 +30,8 @@ use std::os::raw::c_int; #[cfg(feature = "chrono")] use std::ptr; +use super::PyType; + fn ensure_datetime_api(py: Python<'_>) -> PyResult<&'static PyDateTime_CAPI> { if let Some(api) = unsafe { pyo3_ffi::PyDateTimeAPI().as_ref() } { Ok(api) @@ -195,10 +197,15 @@ pub struct PyDate(PyAny); pyobject_native_type!( PyDate, crate::ffi::PyDateTime_Date, - |py| expect_datetime_api(py).DateType, #module=Some("datetime"), #checkfunction=PyDate_Check ); +pyobject_native_type_object_methods!( + PyDate, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).DateType).unbind() + } +); pyobject_subclassable_native_type!(PyDate, crate::ffi::PyDateTime_Date); impl PyDate { @@ -266,10 +273,15 @@ pub struct PyDateTime(PyAny); pyobject_native_type!( PyDateTime, crate::ffi::PyDateTime_DateTime, - |py| expect_datetime_api(py).DateTimeType, #module=Some("datetime"), #checkfunction=PyDateTime_Check ); +pyobject_native_type_object_methods!( + PyDateTime, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).DateTimeType).unbind() + } +); pyobject_subclassable_native_type!(PyDateTime, crate::ffi::PyDateTime_DateTime); impl PyDateTime { @@ -512,10 +524,15 @@ pub struct PyTime(PyAny); pyobject_native_type!( PyTime, crate::ffi::PyDateTime_Time, - |py| expect_datetime_api(py).TimeType, #module=Some("datetime"), #checkfunction=PyTime_Check ); +pyobject_native_type_object_methods!( + PyTime, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).TimeType).unbind() + } +); pyobject_subclassable_native_type!(PyTime, crate::ffi::PyDateTime_Time); impl PyTime { @@ -668,10 +685,15 @@ pub struct PyTzInfo(PyAny); pyobject_native_type!( PyTzInfo, crate::ffi::PyObject, - |py| expect_datetime_api(py).TZInfoType, #module=Some("datetime"), #checkfunction=PyTZInfo_Check ); +pyobject_native_type_object_methods!( + PyTzInfo, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).TZInfoType).unbind() + } +); pyobject_subclassable_native_type!(PyTzInfo, crate::ffi::PyObject); /// Equivalent to `datetime.timezone.utc` @@ -720,10 +742,15 @@ pub struct PyDelta(PyAny); pyobject_native_type!( PyDelta, crate::ffi::PyDateTime_Delta, - |py| expect_datetime_api(py).DeltaType, #module=Some("datetime"), #checkfunction=PyDelta_Check ); +pyobject_native_type_object_methods!( + PyDelta, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).DeltaType).unbind() + } +); pyobject_subclassable_native_type!(PyDelta, crate::ffi::PyDateTime_Delta); impl PyDelta { diff --git a/src/types/dict.rs b/src/types/dict.rs index 129f32dc9e1..c98d223bd2e 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -18,12 +18,8 @@ pub struct PyDict(PyAny); pyobject_subclassable_native_type!(PyDict, crate::ffi::PyDictObject); -pyobject_native_type!( - PyDict, - ffi::PyDictObject, - pyobject_native_static_type_object!(ffi::PyDict_Type), - #checkfunction=ffi::PyDict_Check -); +pyobject_native_type!(PyDict, ffi::PyDictObject, #checkfunction=ffi::PyDict_Check); +pyobject_native_type_object_methods!(PyDict, #global=ffi::PyDict_Type); /// Represents a Python `dict_keys`. #[cfg(not(any(PyPy, GraalPy)))] @@ -31,11 +27,9 @@ pyobject_native_type!( pub struct PyDictKeys(PyAny); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type_core!( - PyDictKeys, - pyobject_native_static_type_object!(ffi::PyDictKeys_Type), - #checkfunction=ffi::PyDictKeys_Check -); +pyobject_native_type_core!(PyDictKeys, #checkfunction=ffi::PyDictKeys_Check); +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PyDictKeys, #global=ffi::PyDictKeys_Type); /// Represents a Python `dict_values`. #[cfg(not(any(PyPy, GraalPy)))] @@ -43,11 +37,9 @@ pyobject_native_type_core!( pub struct PyDictValues(PyAny); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type_core!( - PyDictValues, - pyobject_native_static_type_object!(ffi::PyDictValues_Type), - #checkfunction=ffi::PyDictValues_Check -); +pyobject_native_type_core!(PyDictValues, #checkfunction=ffi::PyDictValues_Check); +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PyDictValues, #global=ffi::PyDictValues_Type); /// Represents a Python `dict_items`. #[cfg(not(any(PyPy, GraalPy)))] @@ -55,11 +47,9 @@ pyobject_native_type_core!( pub struct PyDictItems(PyAny); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type_core!( - PyDictItems, - pyobject_native_static_type_object!(ffi::PyDictItems_Type), - #checkfunction=ffi::PyDictItems_Check -); +pyobject_native_type_core!(PyDictItems, #checkfunction=ffi::PyDictItems_Check); +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PyDictItems, #global=ffi::PyDictItems_Type); impl PyDict { /// Creates a new empty dictionary. diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index 959123ea9db..54e78e556df 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -36,6 +36,10 @@ unsafe impl PyTypeInfo for PyEllipsis { unsafe { ffi::Py_TYPE(ffi::Py_Ellipsis()) } } + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject> { + Some(unsafe { ffi::Py_TYPE(ffi::Py_Ellipsis()) }) + } + #[inline] fn is_type_of(object: &Bound<'_, PyAny>) -> bool { // ellipsis is not usable as a base type diff --git a/src/types/float.rs b/src/types/float.rs index 3c2d6643d18..51b2d150fba 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -26,13 +26,8 @@ use std::os::raw::c_double; pub struct PyFloat(PyAny); pyobject_subclassable_native_type!(PyFloat, crate::ffi::PyFloatObject); - -pyobject_native_type!( - PyFloat, - ffi::PyFloatObject, - pyobject_native_static_type_object!(ffi::PyFloat_Type), - #checkfunction=ffi::PyFloat_Check -); +pyobject_native_type!(PyFloat, ffi::PyFloatObject, #checkfunction=ffi::PyFloat_Check); +pyobject_native_type_object_methods!(PyFloat, #global=ffi::PyFloat_Type); impl PyFloat { /// Creates a new Python `float` object. diff --git a/src/types/frame.rs b/src/types/frame.rs index 8d88d4754ae..3b1528f4f0c 100644 --- a/src/types/frame.rs +++ b/src/types/frame.rs @@ -8,8 +8,5 @@ use crate::PyAny; #[repr(transparent)] pub struct PyFrame(PyAny); -pyobject_native_type_core!( - PyFrame, - pyobject_native_static_type_object!(ffi::PyFrame_Type), - #checkfunction=ffi::PyFrame_Check -); +pyobject_native_type_core!(PyFrame, #checkfunction=ffi::PyFrame_Check); +pyobject_native_type_object_methods!(PyFrame, #global=ffi::PyFrame_Type); diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 3c5a62a01d8..a6c766ba262 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -72,19 +72,11 @@ pub struct PyFrozenSet(PyAny); #[cfg(not(any(PyPy, GraalPy)))] pyobject_subclassable_native_type!(PyFrozenSet, crate::ffi::PySetObject); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type!( - PyFrozenSet, - ffi::PySetObject, - pyobject_native_static_type_object!(ffi::PyFrozenSet_Type), - #checkfunction=ffi::PyFrozenSet_Check -); - +pyobject_native_type!(PyFrozenSet, ffi::PySetObject, #checkfunction=ffi::PyFrozenSet_Check); #[cfg(any(PyPy, GraalPy))] -pyobject_native_type_core!( - PyFrozenSet, - pyobject_native_static_type_object!(ffi::PyFrozenSet_Type), - #checkfunction=ffi::PyFrozenSet_Check -); +pyobject_native_type_core!(PyFrozenSet, #checkfunction=ffi::PyFrozenSet_Check); + +pyobject_native_type_object_methods!(PyFrozenSet, #global=ffi::PyFrozenSet_Type); impl PyFrozenSet { /// Creates a new frozenset. diff --git a/src/types/function.rs b/src/types/function.rs index 039e2774546..9ce6595f2b3 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -18,7 +18,8 @@ use std::ffi::CStr; #[repr(transparent)] pub struct PyCFunction(PyAny); -pyobject_native_type_core!(PyCFunction, pyobject_native_static_type_object!(ffi::PyCFunction_Type), #checkfunction=ffi::PyCFunction_Check); +pyobject_native_type_core!(PyCFunction, #checkfunction=ffi::PyCFunction_Check); +pyobject_native_type_object_methods!(PyCFunction, #global=ffi::PyCFunction_Type); impl PyCFunction { /// Create a new built-in function with keywords (*args and/or **kwargs). @@ -226,4 +227,6 @@ unsafe impl Send for ClosureDestructor {} pub struct PyFunction(PyAny); #[cfg(not(Py_LIMITED_API))] -pyobject_native_type_core!(PyFunction, pyobject_native_static_type_object!(ffi::PyFunction_Type), #checkfunction=ffi::PyFunction_Check); +pyobject_native_type_core!(PyFunction, #checkfunction=ffi::PyFunction_Check); +#[cfg(not(Py_LIMITED_API))] +pyobject_native_type_object_methods!(PyFunction, #global=ffi::PyFunction_Type); diff --git a/src/types/list.rs b/src/types/list.rs index f00c194739f..aae2578b469 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -20,7 +20,8 @@ use crate::types::sequence::PySequenceMethods; #[repr(transparent)] pub struct PyList(PyAny); -pyobject_native_type_core!(PyList, pyobject_native_static_type_object!(ffi::PyList_Type), #checkfunction=ffi::PyList_Check); +pyobject_native_type_core!(PyList, #checkfunction=ffi::PyList_Check); +pyobject_native_type_object_methods!(PyList, #global=ffi::PyList_Type); #[inline] #[track_caller] diff --git a/src/types/mappingproxy.rs b/src/types/mappingproxy.rs index fc28687c561..437986632fe 100644 --- a/src/types/mappingproxy.rs +++ b/src/types/mappingproxy.rs @@ -19,11 +19,8 @@ unsafe fn dict_proxy_check(op: *mut ffi::PyObject) -> c_int { ffi::Py_IS_TYPE(op, std::ptr::addr_of_mut!(ffi::PyDictProxy_Type)) } -pyobject_native_type_core!( - PyMappingProxy, - pyobject_native_static_type_object!(ffi::PyDictProxy_Type), - #checkfunction=dict_proxy_check -); +pyobject_native_type_core!(PyMappingProxy, #checkfunction=dict_proxy_check); +pyobject_native_type_object_methods!(PyMappingProxy, #global=ffi::PyDictProxy_Type); impl PyMappingProxy { /// Creates a mappingproxy from an object. diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index 81acc5cbb2a..c05b344cf28 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -10,7 +10,8 @@ use crate::{ffi, Bound, PyAny}; #[repr(transparent)] pub struct PyMemoryView(PyAny); -pyobject_native_type_core!(PyMemoryView, pyobject_native_static_type_object!(ffi::PyMemoryView_Type), #checkfunction=ffi::PyMemoryView_Check); +pyobject_native_type_core!(PyMemoryView, #checkfunction=ffi::PyMemoryView_Check); +pyobject_native_type_object_methods!(PyMemoryView, #global=ffi::PyMemoryView_Type); impl PyMemoryView { /// Creates a new Python `memoryview` object from another Python object that diff --git a/src/types/mod.rs b/src/types/mod.rs index 677f25d803f..e38d1dd8c6a 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -139,31 +139,25 @@ macro_rules! pyobject_native_type_named ( }; ); -/// Obtain the address of the given static `PyTypeObject`. -#[doc(hidden)] -#[macro_export] -macro_rules! pyobject_native_static_type_object( - ($typeobject:expr) => { - |_py| { - #[allow(unused_unsafe)] // https://github.com/rust-lang/rust/pull/125834 - unsafe { ::std::ptr::addr_of_mut!($typeobject) } - } - }; -); - #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_info( - ($name:ty, $typeobject:expr, $module:expr, $opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $module:expr, $opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { unsafe impl<$($generics,)*> $crate::type_object::PyTypeInfo for $name { const NAME: &'static str = stringify!($name); const MODULE: ::std::option::Option<&'static str> = $module; const OPAQUE: bool = $opaque; #[inline] - #[allow(clippy::redundant_closure_call)] fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - $typeobject(py) + // provided by pyobject_native_type_object_methods!() + Self::type_object_raw_impl(py) + } + + #[inline] + fn try_get_type_object_raw() -> ::std::option::Option<*mut $crate::ffi::PyTypeObject> { + // provided by pyobject_native_type_object_methods!() + Self::try_get_type_object_raw_impl() } $( @@ -194,16 +188,34 @@ macro_rules! pyobject_native_type_marker( #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_core { - ($name:ty, $typeobject:expr, #module=$module:expr, #opaque=$opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, #module=$module:expr, #opaque=$opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { $crate::pyobject_native_type_named!($name $(;$generics)*); $crate::pyobject_native_type_marker!($name); - $crate::pyobject_native_type_info!($name, $typeobject, $module, $opaque $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_info!( + $name, + $module, + $opaque + $(, #checkfunction=$checkfunction)? + $(;$generics)* + ); }; - ($name:ty, $typeobject:expr, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=$module, #opaque=false $(, #checkfunction=$checkfunction)? $(;$generics)*); + ($name:ty, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!( + $name, + #module=$module, + #opaque=false + $(, #checkfunction=$checkfunction)? + $(;$generics)* + ); }; - ($name:ty, $typeobject:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins"), #opaque=false $(, #checkfunction=$checkfunction)? $(;$generics)*); + ($name:ty $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!( + $name, + #module=::std::option::Option::Some("builtins"), + #opaque=false + $(, #checkfunction=$checkfunction)? + $(;$generics)* + ); }; } @@ -236,14 +248,83 @@ macro_rules! pyobject_native_type_sized { #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type { - ($name:ty, $layout:path, $typeobject:expr $(, #module=$module:expr)? $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject $(, #module=$module)? $(, #checkfunction=$checkfunction)? $(;$generics)*); + ($name:ty, $layout:path $(, #module=$module:expr)? $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!($name $(, #module=$module)? $(, #checkfunction=$checkfunction)? $(;$generics)*); // To prevent inheriting native types with ABI3 #[cfg(not(Py_LIMITED_API))] $crate::pyobject_native_type_sized!($name, $layout $(;$generics)*); }; } +/// Implement methods for obtaining the type object associated with a native type. +/// These methods are referred to in `pyobject_native_type_info` for implementing `PyTypeInfo`. +#[doc(hidden)] +#[macro_export] +macro_rules! pyobject_native_type_object_methods { + // the type object is not known statically and so must be created (once) with the GIL held + ($name:ty, #create=$create_type_object:expr) => { + impl $name { + fn type_object_cell() -> &'static $crate::sync::GILOnceCell<$crate::Py<$crate::types::PyType>> { + static TYPE_OBJECT: $crate::sync::GILOnceCell<$crate::Py<$crate::types::PyType>> = + $crate::sync::GILOnceCell::new(); + &TYPE_OBJECT + } + + #[allow(clippy::redundant_closure_call)] + fn type_object_raw_impl(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { + Self::type_object_cell() + .get_or_init(py, || $create_type_object(py)) + .as_ptr() + .cast::<$crate::ffi::PyTypeObject>() + } + + fn try_get_type_object_raw_impl() -> ::std::option::Option<*mut $crate::ffi::PyTypeObject> { + unsafe { + Self::type_object_cell().get_raw().map(|obj| { (*obj).as_ptr().cast() }) + } + } + } + }; + // the type object can be created without holding the GIL + ($name:ty, #get=$get_type_object:expr) => { + impl $name { + fn type_object_raw_impl(_py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { + Self::try_get_type_object_raw_impl().expect("type object is None when it should be Some") + } + + #[allow(clippy::redundant_closure_call)] + fn try_get_type_object_raw_impl() -> ::std::option::Option<*mut $crate::ffi::PyTypeObject> { + Some($get_type_object()) + } + } + }; + // the type object is imported from a module + ($name:ty, #import_module=$import_module:expr, #import_name=$import_name:expr) => { + $crate::pyobject_native_type_object_methods!($name, #create=|py: $crate::Python<'_>| { + let module = stringify!($import_module); + let name = stringify!($import_name); + || -> $crate::PyResult<$crate::Py<$crate::types::PyType>> { + use $crate::types::PyAnyMethods; + $crate::PyResult::Ok(py.import(module)?.getattr(name)?.downcast_into()?.unbind()) + }() + .unwrap_or_else(|e| ::std::panic!("failed to import {}.{}: {}", module, name, e)) + }); + }; + // the type object is known statically + ($name:ty, #global=$ffi_type_object:path) => { + $crate::pyobject_native_type_object_methods!($name, #get=|| { + #[allow(unused_unsafe)] // https://github.com/rust-lang/rust/pull/125834 + unsafe { ::std::ptr::addr_of_mut!($ffi_type_object) } + }); + }; + // the type object is known statically + ($name:ty, #global_ptr=$ffi_type_object:path) => { + $crate::pyobject_native_type_object_methods!($name, #get=|| { + unsafe { $ffi_type_object.cast::<$crate::ffi::PyTypeObject>() } + }); + }; +} + pub(crate) mod any; pub(crate) mod boolobject; pub(crate) mod bytearray; diff --git a/src/types/module.rs b/src/types/module.rs index f3490385721..dd6f6f2e121 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -27,7 +27,8 @@ use std::str; #[repr(transparent)] pub struct PyModule(PyAny); -pyobject_native_type_core!(PyModule, pyobject_native_static_type_object!(ffi::PyModule_Type), #checkfunction=ffi::PyModule_Check); +pyobject_native_type_core!(PyModule, #checkfunction=ffi::PyModule_Check); +pyobject_native_type_object_methods!(PyModule, #global=ffi::PyModule_Type); impl PyModule { /// Creates a new module object with the `__name__` attribute set to `name`. diff --git a/src/types/none.rs b/src/types/none.rs index 14eb050a94c..9b66cb8e482 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -36,6 +36,10 @@ unsafe impl PyTypeInfo for PyNone { unsafe { ffi::Py_TYPE(ffi::Py_None()) } } + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject> { + Some(unsafe { ffi::Py_TYPE(ffi::Py_None()) }) + } + #[inline] fn is_type_of(object: &Bound<'_, PyAny>) -> bool { // NoneType is not usable as a base type diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index e8f1dd94ef0..9c0371cb998 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -40,6 +40,10 @@ unsafe impl PyTypeInfo for PyNotImplemented { unsafe { ffi::Py_TYPE(ffi::Py_NotImplemented()) } } + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject> { + Some(unsafe { ffi::Py_TYPE(ffi::Py_NotImplemented()) }) + } + #[inline] fn is_type_of(object: &Bound<'_, PyAny>) -> bool { // NotImplementedType is not usable as a base type diff --git a/src/types/num.rs b/src/types/num.rs index 0e377f66d48..b31dbe74892 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -14,7 +14,8 @@ use crate::{ffi, instance::Bound, PyAny}; #[repr(transparent)] pub struct PyInt(PyAny); -pyobject_native_type_core!(PyInt, pyobject_native_static_type_object!(ffi::PyLong_Type), #checkfunction=ffi::PyLong_Check); +pyobject_native_type_core!(PyInt, #checkfunction=ffi::PyLong_Check); +pyobject_native_type_object_methods!(PyInt, #global=ffi::PyLong_Type); /// Deprecated alias for [`PyInt`]. #[deprecated(since = "0.23.0", note = "use `PyInt` instead")] diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 81db8cea869..c9de1a3a189 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -11,10 +11,8 @@ use crate::{PyAny, PyResult}; #[repr(transparent)] pub struct PySuper(PyAny); -pyobject_native_type_core!( - PySuper, - pyobject_native_static_type_object!(ffi::PySuper_Type) -); +pyobject_native_type_core!(PySuper); +pyobject_native_type_object_methods!(PySuper, #global=ffi::PySuper_Type); impl PySuper { /// Constructs a new super object. More read about super object: [docs](https://docs.python.org/3/library/functions.html#super) diff --git a/src/types/set.rs b/src/types/set.rs index e7c24f5b1ea..69a7bede420 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -26,19 +26,12 @@ pub struct PySet(PyAny); pyobject_subclassable_native_type!(PySet, crate::ffi::PySetObject); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type!( - PySet, - ffi::PySetObject, - pyobject_native_static_type_object!(ffi::PySet_Type), - #checkfunction=ffi::PySet_Check -); - +pyobject_native_type!(PySet, ffi::PySetObject, #checkfunction=ffi::PySet_Check); #[cfg(any(PyPy, GraalPy))] -pyobject_native_type_core!( - PySet, - pyobject_native_static_type_object!(ffi::PySet_Type), - #checkfunction=ffi::PySet_Check -); +pyobject_native_type_core!(PySet, #checkfunction=ffi::PySet_Check); + +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PySet, #global=ffi::PySet_Type); impl PySet { /// Creates a new set with elements from the given slice. diff --git a/src/types/slice.rs b/src/types/slice.rs index 9ca2aa4ec43..a0ec89d549d 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -19,12 +19,8 @@ use std::convert::Infallible; #[repr(transparent)] pub struct PySlice(PyAny); -pyobject_native_type!( - PySlice, - ffi::PySliceObject, - pyobject_native_static_type_object!(ffi::PySlice_Type), - #checkfunction=ffi::PySlice_Check -); +pyobject_native_type!(PySlice, ffi::PySliceObject, #checkfunction=ffi::PySlice_Check); +pyobject_native_type_object_methods!(PySlice, #global=ffi::PySlice_Type); /// Return value from [`PySliceMethods::indices`]. #[derive(Debug, Eq, PartialEq)] diff --git a/src/types/string.rs b/src/types/string.rs index 65a9e85fa3e..f6c4143c160 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -158,7 +158,8 @@ impl<'a> PyStringData<'a> { #[repr(transparent)] pub struct PyString(PyAny); -pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::PyUnicode_Type), #checkfunction=ffi::PyUnicode_Check); +pyobject_native_type_core!(PyString, #checkfunction=ffi::PyUnicode_Check); +pyobject_native_type_object_methods!(PyString, #global=ffi::PyUnicode_Type); impl PyString { /// Creates a new Python string object. diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 885c0f67031..32d791b599b 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -12,11 +12,8 @@ use crate::{ffi, Bound, PyAny}; #[repr(transparent)] pub struct PyTraceback(PyAny); -pyobject_native_type_core!( - PyTraceback, - pyobject_native_static_type_object!(ffi::PyTraceBack_Type), - #checkfunction=ffi::PyTraceBack_Check -); +pyobject_native_type_core!(PyTraceback, #checkfunction=ffi::PyTraceBack_Check); +pyobject_native_type_object_methods!(PyTraceback, #global=ffi::PyTraceBack_Type); /// Implementation of functionality for [`PyTraceback`]. /// diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 3a1f92815c2..ddff2085fdb 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -60,7 +60,8 @@ fn try_new_from_iter<'py>( #[repr(transparent)] pub struct PyTuple(PyAny); -pyobject_native_type_core!(PyTuple, pyobject_native_static_type_object!(ffi::PyTuple_Type), #checkfunction=ffi::PyTuple_Check); +pyobject_native_type_core!(PyTuple, #checkfunction=ffi::PyTuple_Check); +pyobject_native_type_object_methods!(PyTuple, #global=ffi::PyTuple_Type); impl PyTuple { /// Constructs a new tuple with the given elements. diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 915f37c655b..faa792eb80f 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -19,11 +19,11 @@ pub struct PyType(PyAny); pyobject_native_type_core!( PyType, - pyobject_native_static_type_object!(ffi::PyType_Type), - #module=::std::option::Option::Some("builtins"), + #module=Some("builtins"), #opaque=true, - #checkfunction = ffi::PyType_Check + #checkfunction=ffi::PyType_Check ); +pyobject_native_type_object_methods!(PyType, #global=ffi::PyType_Type); impl crate::impl_::pyclass::PyClassBaseType for PyType { type StaticLayout = crate::impl_::pycell::InvalidStaticLayout; diff --git a/src/types/weakref/reference.rs b/src/types/weakref/reference.rs index dc7ea4a272a..2ff5c1aa574 100644 --- a/src/types/weakref/reference.rs +++ b/src/types/weakref/reference.rs @@ -22,10 +22,11 @@ pyobject_subclassable_native_type!(PyWeakrefReference, crate::ffi::PyWeakReferen pyobject_native_type!( PyWeakrefReference, ffi::PyWeakReference, - pyobject_native_static_type_object!(ffi::_PyWeakref_RefType), #module=Some("weakref"), #checkfunction=ffi::PyWeakref_CheckRefExact ); +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] +pyobject_native_type_object_methods!(PyWeakrefReference, #global=ffi::_PyWeakref_RefType); // When targetting alternative or multiple interpreters, it is better to not use the internal API. #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] From d8d1d2bb4a8d0129af0717a6d75552e98430b571 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 10 Nov 2024 23:51:38 +0000 Subject: [PATCH 32/41] replace Py_TYPE with actual desired type object --- src/impl_/pyclass.rs | 16 +- src/impl_/pyclass_init.rs | 12 +- src/impl_/pymethods.rs | 25 ++- src/impl_/trampoline.rs | 4 +- src/instance.rs | 45 ++++- src/pycell.rs | 46 ++++-- src/pycell/borrow_checker.rs | 13 +- src/pycell/layout.rs | 155 ++++++++++++++---- src/pyclass_init.rs | 4 +- tests/test_class_basics.rs | 6 +- tests/ui/init_without_default.stderr | 4 +- tests/ui/invalid_frozen_pyclass_borrow.rs | 4 +- tests/ui/invalid_frozen_pyclass_borrow.stderr | 20 +-- 13 files changed, 256 insertions(+), 98 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index d51f93bce7e..b166255c68d 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -9,7 +9,7 @@ use crate::{ pymethods::{PyGetterDef, PyMethodDefType}, }, pycell::{ - layout::{PyObjectLayout, PyObjectRecursiveOperations}, + layout::{LazyTypeProvider, PyObjectLayout, PyObjectRecursiveOperations}, PyBorrowError, }, type_object::PyLayout, @@ -1537,7 +1537,7 @@ unsafe fn ensure_no_mutable_alias<'py, ClassT: PyClass>( /// calculates the field pointer from an PyObject pointer #[inline] -fn field_from_object(obj: *mut ffi::PyObject) -> *mut FieldT +fn field_from_object(py: Python<'_>, obj: *mut ffi::PyObject) -> *mut FieldT where ClassT: PyClass, Offset: OffsetCalculator, @@ -1547,7 +1547,9 @@ where #[cfg(Py_3_12)] PyObjectOffset::Relative(offset) => { // Safety: obj must be a valid `PyObject` whose type is a subtype of `ClassT` - let contents = unsafe { PyObjectLayout::get_contents_ptr::(obj) }; + let contents = unsafe { + PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)) + }; (contents.cast::(), offset) } #[cfg(not(Py_3_12))] @@ -1569,7 +1571,7 @@ fn pyo3_get_value_topyobject< obj: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1586,7 +1588,7 @@ where Offset: OffsetCalculator, { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1606,7 +1608,7 @@ where Offset: OffsetCalculator, { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1627,7 +1629,7 @@ fn pyo3_get_value< obj: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index a9983515287..7c7f29839d7 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -1,16 +1,18 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; -use crate::impl_::pyclass::PyClassImpl; use crate::internal::get_slot::TP_ALLOC; -use crate::pycell::layout::{PyClassObjectContents, PyObjectLayout}; +use crate::pycell::layout::{LazyTypeProvider, PyClassObjectContents, PyObjectLayout}; use crate::types::PyType; -use crate::{ffi, Borrowed, PyErr, PyResult, Python}; +use crate::{ffi, Borrowed, PyClass, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; use std::marker::PhantomData; -pub unsafe fn initialize_with_default(obj: *mut ffi::PyObject) { +pub unsafe fn initialize_with_default( + py: Python<'_>, + obj: *mut ffi::PyObject, +) { std::ptr::write( - PyObjectLayout::get_contents_ptr::(obj), + PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)), PyClassObjectContents::new(T::default()), ); } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 66ae6948ad6..88378c3fa5c 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -3,8 +3,8 @@ use crate::gil::LockGIL; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; -use crate::pycell::borrow_checker::PyClassBorrowChecker; -use crate::pycell::layout::PyObjectLayout; +use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; +use crate::pycell::layout::{AssumeInitializedTypeProvider, PyObjectLayout}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; @@ -20,6 +20,7 @@ use std::os::raw::{c_int, c_void}; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr::null_mut; +use super::pycell::{PyClassRecursiveOperations, PyObjectRecursiveOperations}; use super::pyclass::PyClassImpl; use super::trampoline; @@ -285,6 +286,16 @@ pub unsafe fn _call_traverse( where T: PyClass, { + { + let py = Python::assume_gil_acquired(); + // allows functions that traverse the contents of `slf` below to assume that all + // the type objects are initialized. Only classes using the opaque layout use + // type objects to traverse the object. + if T::OPAQUE { + PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + } + } + // It is important the implementation of `__traverse__` cannot safely access the GIL, // c.f. https://github.com/PyO3/pyo3/issues/3165, and hence we do not expose our GIL // token to the user code and lock safe methods for acquiring the GIL. @@ -307,13 +318,13 @@ where let retval = // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so // do not traverse them if not on their owning thread :( - if PyObjectLayout::check_threadsafe::(raw_obj).is_ok() + if PyClassRecursiveOperations::::check_threadsafe(raw_obj).is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread - && PyObjectLayout::get_borrow_checker::(raw_obj).try_borrow().is_ok() { + && T::PyClassMutability::borrow_checker(raw_obj).try_borrow().is_ok() { struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); impl Drop for TraverseGuard<'_, Cls> { fn drop(&mut self) { - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(self.0) }; + let borrow_checker = Cls::PyClassMutability::borrow_checker(self.0); borrow_checker.release_borrow(); } } @@ -321,7 +332,9 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); - let instance = PyObjectLayout::get_data::(raw_obj); + // Safety: type object is manually initialized above + let type_provider = AssumeInitializedTypeProvider::new(); + let instance = PyObjectLayout::get_data::(raw_obj, type_provider); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 52726cfaf03..3af8eaa4a07 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -130,7 +130,7 @@ pub unsafe fn initproc( args: *mut ffi::PyObject, kwargs: *mut ffi::PyObject, // initializes the object to a valid state before running the user-defined init function - initialize: unsafe fn(*mut ffi::PyObject), + initialize: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject), f: for<'py> unsafe fn( Python<'py>, *mut ffi::PyObject, @@ -140,7 +140,7 @@ pub unsafe fn initproc( ) -> c_int { // the map() discards the success value of `f` and converts to the success return value for tp_init (0) trampoline(|py| { - initialize(slf); + initialize(py, slf); f(py, slf, args, kwargs).map(|_| 0) }) } diff --git a/src/instance.rs b/src/instance.rs index d23a5efd598..a9683601d7a 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,6 +1,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::internal_tricks::ptr_from_ref; +use crate::pycell::layout::AssumeInitializedTypeProvider; use crate::pycell::{layout::PyObjectLayout, PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; @@ -325,10 +326,25 @@ where PyRefMut::try_borrow(self) } + /// Call this function before using `get()` + pub fn enable_get(py: Python<'_>) + where + T: PyTypeInfo, + { + // only classes using the opaque layout require type objects for traversal + if T::OPAQUE { + let _ = T::type_object_raw(py); + } + } + /// Provide an immutable borrow of the value `T` without acquiring the GIL. /// /// This is available if the class is [`frozen`][macro@crate::pyclass] and [`Sync`]. /// + /// # Safety + /// + /// `enable_get()` must have been called for `T` beforehand. + /// /// # Examples /// /// ``` @@ -349,7 +365,7 @@ where /// }); /// ``` #[inline] - pub fn get(&self) -> &T + pub unsafe fn get(&self) -> &T where T: PyClass + Sync, { @@ -1281,10 +1297,25 @@ where self.bind(py).try_borrow_mut() } + /// Call this function before using `get()` + pub fn enable_get(py: Python<'_>) + where + T: PyTypeInfo, + { + // only classes using the opaque layout require type objects for traversal + if T::OPAQUE { + let _ = T::type_object_raw(py); + } + } + /// Provide an immutable borrow of the value `T` without acquiring the GIL. /// /// This is available if the class is [`frozen`][macro@crate::pyclass] and [`Sync`]. /// + /// # Safety + /// + /// `enable_get()` must have been called for `T` beforehand. + /// /// # Examples /// /// ``` @@ -1306,12 +1337,14 @@ where /// # Python::with_gil(move |_py| drop(cell)); /// ``` #[inline] - pub fn get(&self) -> &T + pub unsafe fn get(&self) -> &T where T: PyClass + Sync, { + // Safety: `enable_get()` has already been called. + let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; // Safety: The class itself is frozen and `Sync` - unsafe { PyObjectLayout::get_data::(self.as_raw_ref()) } + unsafe { PyObjectLayout::get_data::(self.as_raw_ref(), type_provider) } } } @@ -2347,11 +2380,11 @@ a = A() #[test] fn test_frozen_get() { Python::with_gil(|py| { + Py::::enable_get(py); for i in 0..10 { let instance = Py::new(py, FrozenClass(i)).unwrap(); - assert_eq!(instance.get().0, i); - - assert_eq!(instance.bind(py).get().0, i); + assert_eq!(unsafe { instance.get().0 }, i); + assert_eq!(unsafe { instance.bind(py).get().0 }, i); } }) } diff --git a/src/pycell.rs b/src/pycell.rs index 87204058a14..2ca5fe581c1 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -210,7 +210,7 @@ use std::ops::{Deref, DerefMut}; pub(crate) mod borrow_checker; pub(crate) mod layout; use borrow_checker::PyClassBorrowChecker; -use layout::PyObjectLayout; +use layout::{LazyTypeProvider, PyObjectLayout}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// @@ -311,8 +311,9 @@ impl<'py, T: PyClass> PyRef<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let raw_obj = obj.as_raw_ref(); - unsafe { PyObjectLayout::ensure_threadsafe::(raw_obj) }; - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; borrow_checker .try_borrow() .map(|_| Self { inner: obj.clone() }) @@ -434,19 +435,23 @@ where } } -impl Deref for PyRef<'_, T> { +impl<'py, T: PyClass> Deref for PyRef<'py, T> { type Target = T; #[inline] fn deref(&self) -> &T { - unsafe { PyObjectLayout::get_data::(self.inner.as_raw_ref()) } + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { + PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) + } } } -impl Drop for PyRef<'_, T> { +impl<'py, T: PyClass> Drop for PyRef<'py, T> { fn drop(&mut self) { let obj = self.inner.as_raw_ref(); - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(obj) }; + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; borrow_checker.release_borrow(); } } @@ -567,8 +572,9 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let raw_obj = obj.as_raw_ref(); - unsafe { PyObjectLayout::ensure_threadsafe::(raw_obj) }; - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(raw_obj) }; + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; borrow_checker .try_borrow_mut() .map(|_| Self { inner: obj.clone() }) @@ -618,26 +624,36 @@ where } } -impl> Deref for PyRefMut<'_, T> { +impl<'py, T: PyClass> Deref for PyRefMut<'py, T> { type Target = T; #[inline] fn deref(&self) -> &T { - unsafe { PyObjectLayout::get_data::(self.inner.as_raw_ref()) } + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { + PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) + } } } -impl> DerefMut for PyRefMut<'_, T> { +impl<'py, T: PyClass> DerefMut for PyRefMut<'py, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *PyObjectLayout::get_data_ptr::(self.inner.as_ptr()) } + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { + &mut *PyObjectLayout::get_data_ptr::( + self.inner.as_ptr(), + LazyTypeProvider::new(py), + ) + } } } -impl> Drop for PyRefMut<'_, T> { +impl<'py, T: PyClass> Drop for PyRefMut<'py, T> { fn drop(&mut self) { let obj = self.inner.get_raw_object(); - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(obj) }; + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; borrow_checker.release_borrow_mut(); } } diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index 926ec609e36..bc7ee48604c 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -4,9 +4,10 @@ use std::marker::PhantomData; use std::sync::atomic::{AtomicUsize, Ordering}; -use crate::ffi; use crate::impl_::pyclass::PyClassImpl; +use crate::{ffi, PyTypeInfo}; +use super::layout::AssumeInitializedTypeProvider; use super::{PyBorrowError, PyBorrowMutError, PyObjectLayout}; pub trait PyClassMutability { @@ -171,16 +172,18 @@ pub trait GetBorrowChecker { -> &::Checker; } -impl> GetBorrowChecker for MutableClass { +impl + PyTypeInfo> GetBorrowChecker for MutableClass { fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { - let contents = unsafe { PyObjectLayout::get_contents::(obj) }; + let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; + let contents = unsafe { PyObjectLayout::get_contents::(obj, type_provider) }; &contents.borrow_checker } } -impl> GetBorrowChecker for ImmutableClass { +impl + PyTypeInfo> GetBorrowChecker for ImmutableClass { fn borrow_checker(obj: &ffi::PyObject) -> &EmptySlot { - let contents = unsafe { PyObjectLayout::get_contents::(obj) }; + let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; + let contents = unsafe { PyObjectLayout::get_contents::(obj, type_provider) }; &contents.borrow_checker } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 03727f234c8..80a731f9d08 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -7,11 +7,10 @@ use std::mem::ManuallyDrop; use std::ptr::addr_of_mut; use crate::impl_::pyclass::{ - PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, - PyClassWeakRef, PyObjectOffset, + PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, }; -use crate::pycell::borrow_checker::{PyClassBorrowChecker, GetBorrowChecker}; use crate::internal::get_slot::TP_FREE; +use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; use crate::type_object::PyNativeType; use crate::types::PyType; use crate::{ffi, PyTypeInfo, Python}; @@ -68,8 +67,24 @@ impl PyClassObjectContents { /// then calling a method on a PyObject of type `B` will call the method for `B`, then `A`, then `PyDict`. #[doc(hidden)] pub trait PyObjectRecursiveOperations { + /// `PyTypeInfo::type_object_raw()` may create type objects lazily. + /// This method ensures that the type objects for all ancestor types of the provided object. + unsafe fn ensure_type_objects_initialized(py: Python<'_>); + + /// Call `PyClassThreadChecker::ensure` on all ancestor types of the provided object. + /// + /// # Safety + /// + /// - if the object uses the opaque layout, all ancestor types must be initialized beforehand. unsafe fn ensure_threadsafe(obj: &ffi::PyObject); + + /// Call `PyClassThreadChecker::check` on all ancestor types of the provided object. + /// + /// # Safety + /// + /// - if the object uses the opaque layout, all ancestor types must be initialized beforehand. unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError>; + /// Cleanup then free the memory for `obj`. /// /// # Safety @@ -81,15 +96,22 @@ pub trait PyObjectRecursiveOperations { /// Used to fill out `PyClassBaseType::RecursiveOperations` for instances of `PyClass` pub struct PyClassRecursiveOperations(PhantomData); -impl PyObjectRecursiveOperations for PyClassRecursiveOperations { +impl PyObjectRecursiveOperations for PyClassRecursiveOperations { + unsafe fn ensure_type_objects_initialized(py: Python<'_>) { + let _ = ::type_object_raw(py); + ::RecursiveOperations::ensure_type_objects_initialized(py); + } + unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { - let contents = PyObjectLayout::get_contents::(obj); + let type_provider = AssumeInitializedTypeProvider::new(); + let contents = PyObjectLayout::get_contents::(obj, type_provider); contents.thread_checker.ensure(); ::RecursiveOperations::ensure_threadsafe(obj); } unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError> { - let contents = PyObjectLayout::get_contents::(obj); + let type_provider = AssumeInitializedTypeProvider::new(); + let contents = PyObjectLayout::get_contents::(obj, type_provider); if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } @@ -98,7 +120,8 @@ impl PyObjectRecursiveOperations for PyClassRecursiveOperations< unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. - let contents = &mut *PyObjectLayout::get_contents_ptr::(obj); + let contents = + &mut *PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)); contents.dealloc(py, obj); ::RecursiveOperations::deallocate(py, obj); } @@ -110,6 +133,10 @@ pub struct PyNativeTypeRecursiveOperations(PhantomData); impl PyObjectRecursiveOperations for PyNativeTypeRecursiveOperations { + unsafe fn ensure_type_objects_initialized(py: Python<'_>) { + let _ = ::type_object_raw(py); + } + unsafe fn ensure_threadsafe(_obj: &ffi::PyObject) {} unsafe fn check_threadsafe(_obj: &ffi::PyObject) -> Result<(), PyBorrowError> { @@ -176,18 +203,18 @@ impl PyObjectRecursiveOperations #[doc(hidden)] pub(crate) mod opaque_layout { use super::PyClassObjectContents; - use crate::ffi; - use crate::impl_::pyclass::PyClassImpl; + use super::TypeObjectProvider; + use crate::{ffi, impl_::pyclass::PyClassImpl, PyTypeInfo}; #[cfg(Py_3_12)] - pub fn get_contents_ptr( + pub fn get_contents_ptr>( obj: *mut ffi::PyObject, + type_provider: P, ) -> *mut PyClassObjectContents { #[cfg(Py_3_12)] { - // TODO(matt): this needs to be ::type_object_raw(py) - let type_obj = unsafe { ffi::Py_TYPE(obj) }; - assert!(!type_obj.is_null()); + let type_obj = type_provider.get_type_object(); + assert!(!type_obj.is_null(), "type object is NULL"); let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); pointer.cast() @@ -244,6 +271,51 @@ pub(crate) mod static_layout { unsafe impl PyLayout for InvalidStaticLayout {} } +/// A trait for obtaining a `*mut ffi::PyTypeObject` pointer describing `T` for use with `PyObjectLayout` functions. +/// +/// `PyTypeInfo::type_object_raw()` requires the GIL to be held because it may lazily construct the type object. +/// Some situations require that the GIL is not held so `PyObjectLayout` cannot call this method directly. +/// The different solutions to this have different trade-offs so the caller can decide using a `TypeObjectProvider`. +pub trait TypeObjectProvider { + fn get_type_object(&self) -> *mut ffi::PyTypeObject; +} + +/// Hold the GIL and only obtain/construct the type object if required. +/// +/// Since the type object is only required for accessing opaque objects, this option has the best +/// performance but it requires the GIL being held. +pub struct LazyTypeProvider<'py, T: PyTypeInfo>(PhantomData<&'py T>); +impl<'py, T: PyTypeInfo> LazyTypeProvider<'py, T> { + pub fn new(_py: Python<'py>) -> Self { + Self(PhantomData) + } +} +impl<'py, T: PyTypeInfo> TypeObjectProvider for LazyTypeProvider<'py, T> { + fn get_type_object(&self) -> *mut ffi::PyTypeObject { + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + T::type_object_raw(py) + } +} + +/// Will assume that `PyTypeInfo::type_object_raw()` has been called so the type object +/// is cached and can be obtained without holding the GIL. +pub struct AssumeInitializedTypeProvider(PhantomData); +impl AssumeInitializedTypeProvider { + pub unsafe fn new() -> Self { + Self(PhantomData) + } +} +impl TypeObjectProvider for AssumeInitializedTypeProvider { + fn get_type_object(&self) -> *mut ffi::PyTypeObject { + T::try_get_type_object_raw().unwrap_or_else(|| { + panic!( + "type object for {} not initialized", + std::any::type_name::() + ) + }) + } +} + /// Functions for working with `PyObject`s pub(crate) struct PyObjectLayout {} @@ -251,12 +323,13 @@ impl PyObjectLayout { /// Obtain a pointer to the contents of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_contents_ptr( + pub(crate) unsafe fn get_contents_ptr>( obj: *mut ffi::PyObject, + type_provider: P, ) -> *mut PyClassObjectContents { debug_assert!(!obj.is_null()); if ::OPAQUE { - opaque_layout::get_contents_ptr(obj) + opaque_layout::get_contents_ptr(obj, type_provider) } else { let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); // indicates `ob_base` has type InvalidBaseLayout @@ -269,44 +342,56 @@ impl PyObjectLayout { } } - pub(crate) unsafe fn get_contents( + pub(crate) unsafe fn get_contents>( obj: &ffi::PyObject, + type_provider: P, ) -> &PyClassObjectContents { - &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut()).cast_const() + &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) + .cast_const() } /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_data_ptr(obj: *mut ffi::PyObject) -> *mut T { - let contents = PyObjectLayout::get_contents_ptr::(obj); + pub(crate) unsafe fn get_data_ptr>( + obj: *mut ffi::PyObject, + type_provider: P, + ) -> *mut T { + let contents = PyObjectLayout::get_contents_ptr::(obj, type_provider); (*contents).value.get() } - pub(crate) unsafe fn get_data(obj: &ffi::PyObject) -> &T { - &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut()) - } - - pub(crate) unsafe fn get_borrow_checker( + pub(crate) unsafe fn get_data>( obj: &ffi::PyObject, - ) -> &::Checker { - T::PyClassMutability::borrow_checker(obj) + type_provider: P, + ) -> &T { + &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) } - pub(crate) unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { - PyClassRecursiveOperations::::ensure_threadsafe(obj) + pub(crate) unsafe fn get_borrow_checker<'o, T: PyClassImpl + PyTypeInfo>( + py: Python<'_>, + obj: &'o ffi::PyObject, + ) -> &'o ::Checker { + if T::OPAQUE { + PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + } + T::PyClassMutability::borrow_checker(obj) } - pub(crate) unsafe fn check_threadsafe( + pub(crate) unsafe fn ensure_threadsafe( + py: Python<'_>, obj: &ffi::PyObject, - ) -> Result<(), PyBorrowError> { - PyClassRecursiveOperations::::check_threadsafe(obj) + ) { + if T::OPAQUE { + PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + } + PyClassRecursiveOperations::::ensure_threadsafe(obj); } /// Clean up then free the memory associated with `obj`. /// /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { unsafe { PyClassRecursiveOperations::::deallocate(py, obj); }; @@ -317,7 +402,10 @@ impl PyObjectLayout { /// Use instead of `deallocate()` if `T` has the `Py_TPFLAGS_HAVE_GC` flag set. /// /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - pub(crate) fn deallocate_with_gc(py: Python<'_>, obj: *mut ffi::PyObject) { + pub(crate) fn deallocate_with_gc( + py: Python<'_>, + obj: *mut ffi::PyObject, + ) { unsafe { // TODO(matt): verify T has flag set #[cfg(not(PyPy))] @@ -413,7 +501,6 @@ fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { value.try_into().expect("value should fit in Py_ssize_t") } - #[cfg(test)] #[cfg(feature = "macros")] mod tests { diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 17928ff9995..91d5b5ab471 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -3,7 +3,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::PyClassBaseType; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; -use crate::pycell::layout::PyObjectLayout; +use crate::pycell::layout::{LazyTypeProvider, PyObjectLayout}; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; use crate::{ffi::PyTypeObject, pycell::layout::PyClassObjectContents}; @@ -167,7 +167,7 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; std::ptr::write( - PyObjectLayout::get_contents_ptr::(obj), + PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)), PyClassObjectContents::new(init), ); diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 92afc2cfd1c..6bdac4a5f95 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -645,18 +645,20 @@ fn access_frozen_class_without_gil() { } let py_counter: Py = Python::with_gil(|py| { + Py::::enable_get(py); + let counter = FrozenCounter { value: AtomicUsize::new(0), }; let cell = Bound::new(py, counter).unwrap(); - cell.get().value.fetch_add(1, Ordering::Relaxed); + unsafe { cell.get().value.fetch_add(1, Ordering::Relaxed) }; cell.into() }); - assert_eq!(py_counter.get().value.load(Ordering::Relaxed), 1); + assert_eq!(unsafe { py_counter.get().value.load(Ordering::Relaxed) }, 1); Python::with_gil(move |_py| drop(py_counter)); } diff --git a/tests/ui/init_without_default.stderr b/tests/ui/init_without_default.stderr index 8c333a3c05f..44bc1ab8cd8 100644 --- a/tests/ui/init_without_default.stderr +++ b/tests/ui/init_without_default.stderr @@ -7,8 +7,8 @@ error[E0277]: the trait bound `MyClass: Default` is not satisfied note: required by a bound in `initialize_with_default` --> src/impl_/pyclass_init.rs | - | pub unsafe fn initialize_with_default(obj: *mut ffi::PyObject) { - | ^^^^^^^ required by this bound in `initialize_with_default` + | pub unsafe fn initialize_with_default( + | ^^^^^^^ required by this bound in `initialize_with_default` help: consider annotating `MyClass` with `#[derive(Default)]` | 4 + #[derive(Default)] diff --git a/tests/ui/invalid_frozen_pyclass_borrow.rs b/tests/ui/invalid_frozen_pyclass_borrow.rs index 6379a8707c5..4846080676d 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.rs +++ b/tests/ui/invalid_frozen_pyclass_borrow.rs @@ -26,11 +26,11 @@ fn borrow_mut_of_child_fails(child: Py, py: Python) { } fn py_get_of_mutable_class_fails(class: Py) { - class.get(); + unsafe { class.get(); } } fn pyclass_get_of_mutable_class_fails(class: &Bound<'_, MutableBase>) { - class.get(); + unsafe { class.get(); } } #[pyclass(frozen)] diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index 52a0623f282..2d2b7deb6ce 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -60,31 +60,31 @@ note: required by a bound in `pyo3::Bound::<'py, T>::borrow_mut` | ^^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:11 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:20 | -29 | class.get(); - | ^^^ expected `True`, found `False` +29 | unsafe { class.get(); } + | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::Py::::get` --> src/instance.rs | - | pub fn get(&self) -> &T - | --- required by a bound in this associated function + | pub unsafe fn get(&self) -> &T + | --- required by a bound in this associated function | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `Py::::get` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:33:11 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:33:20 | -33 | class.get(); - | ^^^ expected `True`, found `False` +33 | unsafe { class.get(); } + | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::Bound::<'py, T>::get` --> src/instance.rs | - | pub fn get(&self) -> &T - | --- required by a bound in this associated function + | pub unsafe fn get(&self) -> &T + | --- required by a bound in this associated function | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::get` From 6a00aa9c64af7c65ca30a0c514a703adc40ecffa Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 11 Nov 2024 00:19:06 +0000 Subject: [PATCH 33/41] remove unsafe from some functions --- src/impl_/pyclass.rs | 3 +-- src/instance.rs | 3 +-- src/pycell.rs | 20 ++++++++------------ src/pycell/borrow_checker.rs | 4 ++-- src/pycell/layout.rs | 35 +++++++++++++++++++++-------------- 5 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index b166255c68d..1fa30e9518c 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1140,8 +1140,7 @@ pub trait PyClassBaseType: Sized { /// A struct that describes the memory layout of a `*mut ffi:PyObject` with the type of `Self`. /// Only valid when `::OPAQUE` is false. type StaticLayout: PyLayout; - // TODO(matt): introduce :PyTypeInfo bounds - type BaseNativeType; + type BaseNativeType: PyTypeInfo; type RecursiveOperations: PyObjectRecursiveOperations; type Initializer: PyObjectInit; type PyClassMutability: PyClassMutability; diff --git a/src/instance.rs b/src/instance.rs index a9683601d7a..1f6680757c4 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1343,8 +1343,7 @@ where { // Safety: `enable_get()` has already been called. let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; - // Safety: The class itself is frozen and `Sync` - unsafe { PyObjectLayout::get_data::(self.as_raw_ref(), type_provider) } + PyObjectLayout::get_data::(self.as_raw_ref(), type_provider) } } diff --git a/src/pycell.rs b/src/pycell.rs index 2ca5fe581c1..9f3852f3490 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -312,8 +312,8 @@ impl<'py, T: PyClass> PyRef<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let raw_obj = obj.as_raw_ref(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; + PyObjectLayout::ensure_threadsafe::(py, raw_obj); + let borrow_checker = PyObjectLayout::get_borrow_checker::(py, raw_obj); borrow_checker .try_borrow() .map(|_| Self { inner: obj.clone() }) @@ -441,9 +441,7 @@ impl<'py, T: PyClass> Deref for PyRef<'py, T> { #[inline] fn deref(&self) -> &T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - unsafe { - PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) - } + PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) } } @@ -451,7 +449,7 @@ impl<'py, T: PyClass> Drop for PyRef<'py, T> { fn drop(&mut self) { let obj = self.inner.as_raw_ref(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; + let borrow_checker = PyObjectLayout::get_borrow_checker::(py, obj); borrow_checker.release_borrow(); } } @@ -573,8 +571,8 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let raw_obj = obj.as_raw_ref(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; + PyObjectLayout::ensure_threadsafe::(py, raw_obj); + let borrow_checker = PyObjectLayout::get_borrow_checker::(py, raw_obj); borrow_checker .try_borrow_mut() .map(|_| Self { inner: obj.clone() }) @@ -630,9 +628,7 @@ impl<'py, T: PyClass> Deref for PyRefMut<'py, T> { #[inline] fn deref(&self) -> &T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - unsafe { - PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) - } + PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) } } @@ -653,7 +649,7 @@ impl<'py, T: PyClass> Drop for PyRefMut<'py, T> { fn drop(&mut self) { let obj = self.inner.get_raw_object(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; + let borrow_checker = PyObjectLayout::get_borrow_checker::(py, obj); borrow_checker.release_borrow_mut(); } } diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index bc7ee48604c..320493c4195 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -175,7 +175,7 @@ pub trait GetBorrowChecker { impl + PyTypeInfo> GetBorrowChecker for MutableClass { fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; - let contents = unsafe { PyObjectLayout::get_contents::(obj, type_provider) }; + let contents = PyObjectLayout::get_contents::(obj, type_provider); &contents.borrow_checker } } @@ -183,7 +183,7 @@ impl + PyTypeInfo> GetBorrowChecker impl + PyTypeInfo> GetBorrowChecker for ImmutableClass { fn borrow_checker(obj: &ffi::PyObject) -> &EmptySlot { let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; - let contents = unsafe { PyObjectLayout::get_contents::(obj, type_provider) }; + let contents = PyObjectLayout::get_contents::(obj, type_provider); &contents.borrow_checker } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 80a731f9d08..c31f2bfde90 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -342,12 +342,14 @@ impl PyObjectLayout { } } - pub(crate) unsafe fn get_contents>( + pub(crate) fn get_contents>( obj: &ffi::PyObject, type_provider: P, ) -> &PyClassObjectContents { - &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) - .cast_const() + unsafe { + &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) + .cast_const() + } } /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. @@ -361,31 +363,37 @@ impl PyObjectLayout { (*contents).value.get() } - pub(crate) unsafe fn get_data>( + pub(crate) fn get_data>( obj: &ffi::PyObject, type_provider: P, ) -> &T { - &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) + unsafe { + &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) + } } - pub(crate) unsafe fn get_borrow_checker<'o, T: PyClassImpl + PyTypeInfo>( + pub(crate) fn get_borrow_checker<'o, T: PyClassImpl + PyTypeInfo>( py: Python<'_>, obj: &'o ffi::PyObject, ) -> &'o ::Checker { - if T::OPAQUE { - PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + unsafe { + if T::OPAQUE { + PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + } + T::PyClassMutability::borrow_checker(obj) } - T::PyClassMutability::borrow_checker(obj) } - pub(crate) unsafe fn ensure_threadsafe( + pub(crate) fn ensure_threadsafe( py: Python<'_>, obj: &ffi::PyObject, ) { - if T::OPAQUE { - PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + unsafe { + if T::OPAQUE { + PyClassRecursiveOperations::::ensure_type_objects_initialized(py); + } + PyClassRecursiveOperations::::ensure_threadsafe(obj); } - PyClassRecursiveOperations::::ensure_threadsafe(obj); } /// Clean up then free the memory associated with `obj`. @@ -407,7 +415,6 @@ impl PyObjectLayout { obj: *mut ffi::PyObject, ) { unsafe { - // TODO(matt): verify T has flag set #[cfg(not(PyPy))] { ffi::PyObject_GC_UnTrack(obj.cast()); From bd4f51a074dfe631862657a4db1036ff7bcff472 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 11 Nov 2024 19:40:21 +0000 Subject: [PATCH 34/41] remove unsafe from some functions where possible --- src/impl_/pymethods.rs | 11 ++-- src/instance.rs | 3 +- src/pycell/borrow_checker.rs | 24 +++++--- src/pycell/layout.rs | 110 ++++++++++++++++++----------------- 4 files changed, 81 insertions(+), 67 deletions(-) diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 88378c3fa5c..078e1f9b5ad 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -315,16 +315,20 @@ where // traversal is running so no mutations can occur. let raw_obj = &*slf; + // SAFETY: type objects for `T` and all base classes of `T` have been initialized + // above if they are required. + let type_provider = AssumeInitializedTypeProvider::new(); + let retval = // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so // do not traverse them if not on their owning thread :( - if PyClassRecursiveOperations::::check_threadsafe(raw_obj).is_ok() + if PyClassRecursiveOperations::::check_threadsafe(raw_obj, type_provider).is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread - && T::PyClassMutability::borrow_checker(raw_obj).try_borrow().is_ok() { + && T::PyClassMutability::borrow_checker(raw_obj, type_provider).try_borrow().is_ok() { struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); impl Drop for TraverseGuard<'_, Cls> { fn drop(&mut self) { - let borrow_checker = Cls::PyClassMutability::borrow_checker(self.0); + let borrow_checker = Cls::PyClassMutability::borrow_checker(self.0, unsafe { AssumeInitializedTypeProvider::new() }); borrow_checker.release_borrow(); } } @@ -333,7 +337,6 @@ where // traversing the object. This allows us to read `instance` safely. let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); // Safety: type object is manually initialized above - let type_provider = AssumeInitializedTypeProvider::new(); let instance = PyObjectLayout::get_data::(raw_obj, type_provider); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index 1f6680757c4..1808a639d34 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -369,6 +369,7 @@ where where T: PyClass + Sync, { + // Safety: `enable_get()` has already been called for `T`. self.1.get() } @@ -1341,7 +1342,7 @@ where where T: PyClass + Sync, { - // Safety: `enable_get()` has already been called. + // Safety: `enable_get()` has already been called for `T`. let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; PyObjectLayout::get_data::(self.as_raw_ref(), type_provider) } diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index 320493c4195..ce954512e7e 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -7,7 +7,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::impl_::pyclass::PyClassImpl; use crate::{ffi, PyTypeInfo}; -use super::layout::AssumeInitializedTypeProvider; +use super::layout::TypeObjectProvider; use super::{PyBorrowError, PyBorrowMutError, PyObjectLayout}; pub trait PyClassMutability { @@ -168,21 +168,24 @@ impl PyClassBorrowChecker for BorrowChecker { } pub trait GetBorrowChecker { - fn borrow_checker(obj: &ffi::PyObject) - -> &::Checker; + fn borrow_checker( + obj: &ffi::PyObject, + type_provider: P, + ) -> &::Checker; } impl + PyTypeInfo> GetBorrowChecker for MutableClass { - fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { - let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; + fn borrow_checker( + obj: &ffi::PyObject, + type_provider: P, + ) -> &BorrowChecker { let contents = PyObjectLayout::get_contents::(obj, type_provider); &contents.borrow_checker } } impl + PyTypeInfo> GetBorrowChecker for ImmutableClass { - fn borrow_checker(obj: &ffi::PyObject) -> &EmptySlot { - let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; + fn borrow_checker(obj: &ffi::PyObject, type_provider: P) -> &EmptySlot { let contents = PyObjectLayout::get_contents::(obj, type_provider); &contents.borrow_checker } @@ -195,9 +198,12 @@ where T::BaseType: PyClassImpl, ::PyClassMutability: PyClassMutability, { - fn borrow_checker(obj: &ffi::PyObject) -> &BorrowChecker { + fn borrow_checker( + obj: &ffi::PyObject, + type_provider: P, + ) -> &BorrowChecker { // the same PyObject pointer can be re-interpreted as the base/parent type - <::PyClassMutability as GetBorrowChecker>::borrow_checker(obj) + <::PyClassMutability as GetBorrowChecker>::borrow_checker(obj, type_provider) } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index c31f2bfde90..9579d30823d 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -72,18 +72,13 @@ pub trait PyObjectRecursiveOperations { unsafe fn ensure_type_objects_initialized(py: Python<'_>); /// Call `PyClassThreadChecker::ensure` on all ancestor types of the provided object. - /// - /// # Safety - /// - /// - if the object uses the opaque layout, all ancestor types must be initialized beforehand. - unsafe fn ensure_threadsafe(obj: &ffi::PyObject); + fn ensure_threadsafe(obj: &ffi::PyObject, type_provider: P); /// Call `PyClassThreadChecker::check` on all ancestor types of the provided object. - /// - /// # Safety - /// - /// - if the object uses the opaque layout, all ancestor types must be initialized beforehand. - unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError>; + fn check_threadsafe( + obj: &ffi::PyObject, + type_provider: P, + ) -> Result<(), PyBorrowError>; /// Cleanup then free the memory for `obj`. /// @@ -102,20 +97,24 @@ impl PyObjectRecursiveOperations for PyClassRecursi ::RecursiveOperations::ensure_type_objects_initialized(py); } - unsafe fn ensure_threadsafe(obj: &ffi::PyObject) { - let type_provider = AssumeInitializedTypeProvider::new(); + fn ensure_threadsafe(obj: &ffi::PyObject, type_provider: P) { let contents = PyObjectLayout::get_contents::(obj, type_provider); contents.thread_checker.ensure(); - ::RecursiveOperations::ensure_threadsafe(obj); + ::RecursiveOperations::ensure_threadsafe( + obj, + type_provider, + ); } - unsafe fn check_threadsafe(obj: &ffi::PyObject) -> Result<(), PyBorrowError> { - let type_provider = AssumeInitializedTypeProvider::new(); + fn check_threadsafe( + obj: &ffi::PyObject, + type_provider: P, + ) -> Result<(), PyBorrowError> { let contents = PyObjectLayout::get_contents::(obj, type_provider); if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } - ::RecursiveOperations::check_threadsafe(obj) + ::RecursiveOperations::check_threadsafe(obj, type_provider) } unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { @@ -137,9 +136,12 @@ impl PyObjectRecursiveOperations let _ = ::type_object_raw(py); } - unsafe fn ensure_threadsafe(_obj: &ffi::PyObject) {} + fn ensure_threadsafe(_obj: &ffi::PyObject, _type_provider: P) {} - unsafe fn check_threadsafe(_obj: &ffi::PyObject) -> Result<(), PyBorrowError> { + fn check_threadsafe( + _obj: &ffi::PyObject, + _type_provider: P, + ) -> Result<(), PyBorrowError> { Ok(()) } @@ -207,13 +209,13 @@ pub(crate) mod opaque_layout { use crate::{ffi, impl_::pyclass::PyClassImpl, PyTypeInfo}; #[cfg(Py_3_12)] - pub fn get_contents_ptr>( + pub fn get_contents_ptr( obj: *mut ffi::PyObject, type_provider: P, ) -> *mut PyClassObjectContents { #[cfg(Py_3_12)] { - let type_obj = type_provider.get_type_object(); + let type_obj = type_provider.get_type_object::(); assert!(!type_obj.is_null(), "type object is NULL"); let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); @@ -276,37 +278,49 @@ pub(crate) mod static_layout { /// `PyTypeInfo::type_object_raw()` requires the GIL to be held because it may lazily construct the type object. /// Some situations require that the GIL is not held so `PyObjectLayout` cannot call this method directly. /// The different solutions to this have different trade-offs so the caller can decide using a `TypeObjectProvider`. -pub trait TypeObjectProvider { - fn get_type_object(&self) -> *mut ffi::PyTypeObject; +pub trait TypeObjectProvider: Clone + Copy { + fn get_type_object(&self) -> *mut ffi::PyTypeObject; } -/// Hold the GIL and only obtain/construct the type object if required. -/// -/// Since the type object is only required for accessing opaque objects, this option has the best -/// performance but it requires the GIL being held. -pub struct LazyTypeProvider<'py, T: PyTypeInfo>(PhantomData<&'py T>); -impl<'py, T: PyTypeInfo> LazyTypeProvider<'py, T> { +/// Hold the GIL and only obtain/construct type objects lazily when required. +#[derive(Clone, Copy)] +pub struct LazyTypeProvider<'py>(PhantomData<&'py ()>); +impl<'py> LazyTypeProvider<'py> { pub fn new(_py: Python<'py>) -> Self { Self(PhantomData) } } -impl<'py, T: PyTypeInfo> TypeObjectProvider for LazyTypeProvider<'py, T> { - fn get_type_object(&self) -> *mut ffi::PyTypeObject { +impl<'py> TypeObjectProvider for LazyTypeProvider<'py> { + fn get_type_object(&self) -> *mut ffi::PyTypeObject { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; T::type_object_raw(py) } } -/// Will assume that `PyTypeInfo::type_object_raw()` has been called so the type object -/// is cached and can be obtained without holding the GIL. -pub struct AssumeInitializedTypeProvider(PhantomData); -impl AssumeInitializedTypeProvider { +/// Assume that `PyTypeInfo::type_object_raw()` has been called for any of the required type objects. +/// +/// once initialized, the type objects are cached and can be obtained without holding the GIL. +#[derive(Clone, Copy)] +pub struct AssumeInitializedTypeProvider; +impl AssumeInitializedTypeProvider { + /// Create a type provider that assumes that whatever `T` it is called with has already been initialized. + /// + /// # Safety + /// + /// - ensure that any `T` that may be used with this object has already been initialized + /// by calling `T::type_object_raw()`. + /// - only `PyTypeInfo::OPAQUE` classes require type objects for traversal so if this object is only + /// used with non-opaque classes then no action is required. + /// - when used with `PyClassRecursiveOperations` or `GetBorrowChecker`, the object may be used with + /// base classes as well as the most derived type. + /// `PyClassRecursiveOperations::ensure_type_objects_initialized()` can be used to initialize + /// all base classes above the given type. pub unsafe fn new() -> Self { - Self(PhantomData) + Self } } -impl TypeObjectProvider for AssumeInitializedTypeProvider { - fn get_type_object(&self) -> *mut ffi::PyTypeObject { +impl TypeObjectProvider for AssumeInitializedTypeProvider { + fn get_type_object(&self) -> *mut ffi::PyTypeObject { T::try_get_type_object_raw().unwrap_or_else(|| { panic!( "type object for {} not initialized", @@ -323,7 +337,7 @@ impl PyObjectLayout { /// Obtain a pointer to the contents of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_contents_ptr>( + pub(crate) unsafe fn get_contents_ptr( obj: *mut ffi::PyObject, type_provider: P, ) -> *mut PyClassObjectContents { @@ -342,7 +356,7 @@ impl PyObjectLayout { } } - pub(crate) fn get_contents>( + pub(crate) fn get_contents( obj: &ffi::PyObject, type_provider: P, ) -> &PyClassObjectContents { @@ -355,7 +369,7 @@ impl PyObjectLayout { /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_data_ptr>( + pub(crate) unsafe fn get_data_ptr( obj: *mut ffi::PyObject, type_provider: P, ) -> *mut T { @@ -363,7 +377,7 @@ impl PyObjectLayout { (*contents).value.get() } - pub(crate) fn get_data>( + pub(crate) fn get_data( obj: &ffi::PyObject, type_provider: P, ) -> &T { @@ -376,24 +390,14 @@ impl PyObjectLayout { py: Python<'_>, obj: &'o ffi::PyObject, ) -> &'o ::Checker { - unsafe { - if T::OPAQUE { - PyClassRecursiveOperations::::ensure_type_objects_initialized(py); - } - T::PyClassMutability::borrow_checker(obj) - } + T::PyClassMutability::borrow_checker(obj, LazyTypeProvider::new(py)) } pub(crate) fn ensure_threadsafe( py: Python<'_>, obj: &ffi::PyObject, ) { - unsafe { - if T::OPAQUE { - PyClassRecursiveOperations::::ensure_type_objects_initialized(py); - } - PyClassRecursiveOperations::::ensure_threadsafe(obj); - } + PyClassRecursiveOperations::::ensure_threadsafe(obj, LazyTypeProvider::new(py)); } /// Clean up then free the memory associated with `obj`. From 122b8386503269207be1abf292d4fb42f470921f Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 11 Nov 2024 20:14:04 +0000 Subject: [PATCH 35/41] replace provider with strategy to reduce use of generics --- src/impl_/pyclass.rs | 4 +- src/impl_/pyclass_init.rs | 5 +- src/impl_/pymethods.rs | 15 ++-- src/instance.rs | 6 +- src/pycell.rs | 10 +-- src/pycell/borrow_checker.rs | 37 +++++---- src/pycell/layout.rs | 151 ++++++++++++++++------------------- src/pyclass_init.rs | 4 +- 8 files changed, 112 insertions(+), 120 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 1fa30e9518c..a0e6c22adb2 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -9,7 +9,7 @@ use crate::{ pymethods::{PyGetterDef, PyMethodDefType}, }, pycell::{ - layout::{LazyTypeProvider, PyObjectLayout, PyObjectRecursiveOperations}, + layout::{PyObjectLayout, PyObjectRecursiveOperations, TypeObjectStrategy}, PyBorrowError, }, type_object::PyLayout, @@ -1547,7 +1547,7 @@ where PyObjectOffset::Relative(offset) => { // Safety: obj must be a valid `PyObject` whose type is a subtype of `ClassT` let contents = unsafe { - PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)) + PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)) }; (contents.cast::(), offset) } diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 7c7f29839d7..91359cdb04b 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -1,7 +1,7 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; use crate::internal::get_slot::TP_ALLOC; -use crate::pycell::layout::{LazyTypeProvider, PyClassObjectContents, PyObjectLayout}; +use crate::pycell::layout::{PyClassObjectContents, PyObjectLayout, TypeObjectStrategy}; use crate::types::PyType; use crate::{ffi, Borrowed, PyClass, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; @@ -11,8 +11,9 @@ pub unsafe fn initialize_with_default( py: Python<'_>, obj: *mut ffi::PyObject, ) { + // TODO(matt): need to initialize all base classes, should be recursive std::ptr::write( - PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)), + PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)), PyClassObjectContents::new(T::default()), ); } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 078e1f9b5ad..88ba51c1fdb 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -4,7 +4,7 @@ use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; -use crate::pycell::layout::{AssumeInitializedTypeProvider, PyObjectLayout}; +use crate::pycell::layout::{PyObjectLayout, TypeObjectStrategy}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; @@ -317,18 +317,21 @@ where // SAFETY: type objects for `T` and all base classes of `T` have been initialized // above if they are required. - let type_provider = AssumeInitializedTypeProvider::new(); + let strategy = TypeObjectStrategy::assume_init(); let retval = // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so // do not traverse them if not on their owning thread :( - if PyClassRecursiveOperations::::check_threadsafe(raw_obj, type_provider).is_ok() + if PyClassRecursiveOperations::::check_threadsafe(raw_obj, strategy).is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread - && T::PyClassMutability::borrow_checker(raw_obj, type_provider).try_borrow().is_ok() { + && T::PyClassMutability::borrow_checker(raw_obj, strategy).try_borrow().is_ok() { struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); impl Drop for TraverseGuard<'_, Cls> { fn drop(&mut self) { - let borrow_checker = Cls::PyClassMutability::borrow_checker(self.0, unsafe { AssumeInitializedTypeProvider::new() }); + let borrow_checker = Cls::PyClassMutability::borrow_checker( + self.0, + unsafe { TypeObjectStrategy::assume_init() } + ); borrow_checker.release_borrow(); } } @@ -337,7 +340,7 @@ where // traversing the object. This allows us to read `instance` safely. let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); // Safety: type object is manually initialized above - let instance = PyObjectLayout::get_data::(raw_obj, type_provider); + let instance = PyObjectLayout::get_data::(raw_obj, strategy); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index 1808a639d34..fd4119452d3 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,7 +1,7 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; use crate::internal_tricks::ptr_from_ref; -use crate::pycell::layout::AssumeInitializedTypeProvider; +use crate::pycell::layout::TypeObjectStrategy; use crate::pycell::{layout::PyObjectLayout, PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; @@ -1343,8 +1343,8 @@ where T: PyClass + Sync, { // Safety: `enable_get()` has already been called for `T`. - let type_provider = unsafe { AssumeInitializedTypeProvider::new() }; - PyObjectLayout::get_data::(self.as_raw_ref(), type_provider) + let strategy = unsafe { TypeObjectStrategy::assume_init() }; + PyObjectLayout::get_data::(self.as_raw_ref(), strategy) } } diff --git a/src/pycell.rs b/src/pycell.rs index 9f3852f3490..4a066d78ee5 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -210,7 +210,7 @@ use std::ops::{Deref, DerefMut}; pub(crate) mod borrow_checker; pub(crate) mod layout; use borrow_checker::PyClassBorrowChecker; -use layout::{LazyTypeProvider, PyObjectLayout}; +use layout::{PyObjectLayout, TypeObjectStrategy}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// @@ -441,7 +441,7 @@ impl<'py, T: PyClass> Deref for PyRef<'py, T> { #[inline] fn deref(&self) -> &T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) + PyObjectLayout::get_data::(self.inner.as_raw_ref(), TypeObjectStrategy::lazy(py)) } } @@ -628,7 +628,7 @@ impl<'py, T: PyClass> Deref for PyRefMut<'py, T> { #[inline] fn deref(&self) -> &T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - PyObjectLayout::get_data::(self.inner.as_raw_ref(), LazyTypeProvider::new(py)) + PyObjectLayout::get_data::(self.inner.as_raw_ref(), TypeObjectStrategy::lazy(py)) } } @@ -637,9 +637,9 @@ impl<'py, T: PyClass> DerefMut for PyRefMut<'py, T> { fn deref_mut(&mut self) -> &mut T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; unsafe { - &mut *PyObjectLayout::get_data_ptr::( + &mut *PyObjectLayout::get_data_ptr::( self.inner.as_ptr(), - LazyTypeProvider::new(py), + TypeObjectStrategy::lazy(py), ) } } diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index ce954512e7e..f63d960116b 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -7,7 +7,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::impl_::pyclass::PyClassImpl; use crate::{ffi, PyTypeInfo}; -use super::layout::TypeObjectProvider; +use super::layout::TypeObjectStrategy; use super::{PyBorrowError, PyBorrowMutError, PyObjectLayout}; pub trait PyClassMutability { @@ -168,25 +168,28 @@ impl PyClassBorrowChecker for BorrowChecker { } pub trait GetBorrowChecker { - fn borrow_checker( - obj: &ffi::PyObject, - type_provider: P, - ) -> &::Checker; + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a ::Checker; } impl + PyTypeInfo> GetBorrowChecker for MutableClass { - fn borrow_checker( - obj: &ffi::PyObject, - type_provider: P, - ) -> &BorrowChecker { - let contents = PyObjectLayout::get_contents::(obj, type_provider); + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a BorrowChecker { + let contents = PyObjectLayout::get_contents::(obj, strategy); &contents.borrow_checker } } impl + PyTypeInfo> GetBorrowChecker for ImmutableClass { - fn borrow_checker(obj: &ffi::PyObject, type_provider: P) -> &EmptySlot { - let contents = PyObjectLayout::get_contents::(obj, type_provider); + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a EmptySlot { + let contents = PyObjectLayout::get_contents::(obj, strategy); &contents.borrow_checker } } @@ -198,12 +201,12 @@ where T::BaseType: PyClassImpl, ::PyClassMutability: PyClassMutability, { - fn borrow_checker( - obj: &ffi::PyObject, - type_provider: P, - ) -> &BorrowChecker { + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a BorrowChecker { // the same PyObject pointer can be re-interpreted as the base/parent type - <::PyClassMutability as GetBorrowChecker>::borrow_checker(obj, type_provider) + <::PyClassMutability as GetBorrowChecker>::borrow_checker(obj, strategy) } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 9579d30823d..206edc7b7d1 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -72,12 +72,12 @@ pub trait PyObjectRecursiveOperations { unsafe fn ensure_type_objects_initialized(py: Python<'_>); /// Call `PyClassThreadChecker::ensure` on all ancestor types of the provided object. - fn ensure_threadsafe(obj: &ffi::PyObject, type_provider: P); + fn ensure_threadsafe(obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>); /// Call `PyClassThreadChecker::check` on all ancestor types of the provided object. - fn check_threadsafe( + fn check_threadsafe( obj: &ffi::PyObject, - type_provider: P, + strategy: TypeObjectStrategy<'_>, ) -> Result<(), PyBorrowError>; /// Cleanup then free the memory for `obj`. @@ -97,30 +97,27 @@ impl PyObjectRecursiveOperations for PyClassRecursi ::RecursiveOperations::ensure_type_objects_initialized(py); } - fn ensure_threadsafe(obj: &ffi::PyObject, type_provider: P) { - let contents = PyObjectLayout::get_contents::(obj, type_provider); + fn ensure_threadsafe(obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>) { + let contents = PyObjectLayout::get_contents::(obj, strategy); contents.thread_checker.ensure(); - ::RecursiveOperations::ensure_threadsafe( - obj, - type_provider, - ); + ::RecursiveOperations::ensure_threadsafe(obj, strategy); } - fn check_threadsafe( + fn check_threadsafe( obj: &ffi::PyObject, - type_provider: P, + strategy: TypeObjectStrategy<'_>, ) -> Result<(), PyBorrowError> { - let contents = PyObjectLayout::get_contents::(obj, type_provider); + let contents = PyObjectLayout::get_contents::(obj, strategy); if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } - ::RecursiveOperations::check_threadsafe(obj, type_provider) + ::RecursiveOperations::check_threadsafe(obj, strategy) } unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { // Safety: Python only calls tp_dealloc when no references to the object remain. let contents = - &mut *PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)); + &mut *PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)); contents.dealloc(py, obj); ::RecursiveOperations::deallocate(py, obj); } @@ -136,11 +133,11 @@ impl PyObjectRecursiveOperations let _ = ::type_object_raw(py); } - fn ensure_threadsafe(_obj: &ffi::PyObject, _type_provider: P) {} + fn ensure_threadsafe(_obj: &ffi::PyObject, _strategy: TypeObjectStrategy<'_>) {} - fn check_threadsafe( + fn check_threadsafe( _obj: &ffi::PyObject, - _type_provider: P, + _strategy: TypeObjectStrategy<'_>, ) -> Result<(), PyBorrowError> { Ok(()) } @@ -205,17 +202,27 @@ impl PyObjectRecursiveOperations #[doc(hidden)] pub(crate) mod opaque_layout { use super::PyClassObjectContents; - use super::TypeObjectProvider; + use super::TypeObjectStrategy; use crate::{ffi, impl_::pyclass::PyClassImpl, PyTypeInfo}; #[cfg(Py_3_12)] - pub fn get_contents_ptr( + pub fn get_contents_ptr( obj: *mut ffi::PyObject, - type_provider: P, + strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { #[cfg(Py_3_12)] { - let type_obj = type_provider.get_type_object::(); + let type_obj = match strategy { + TypeObjectStrategy::Lazy(py) => T::type_object_raw(py), + TypeObjectStrategy::AssumeInit(_) => { + T::try_get_type_object_raw().unwrap_or_else(|| { + panic!( + "type object for {} not initialized", + std::any::type_name::() + ) + }) + } + }; assert!(!type_obj.is_null(), "type object is NULL"); let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); @@ -273,60 +280,40 @@ pub(crate) mod static_layout { unsafe impl PyLayout for InvalidStaticLayout {} } -/// A trait for obtaining a `*mut ffi::PyTypeObject` pointer describing `T` for use with `PyObjectLayout` functions. +/// The method to use for obtaining a `*mut ffi::PyTypeObject` pointer describing `T: PyTypeInfo` for +/// use with `PyObjectLayout` functions. /// /// `PyTypeInfo::type_object_raw()` requires the GIL to be held because it may lazily construct the type object. /// Some situations require that the GIL is not held so `PyObjectLayout` cannot call this method directly. -/// The different solutions to this have different trade-offs so the caller can decide using a `TypeObjectProvider`. -pub trait TypeObjectProvider: Clone + Copy { - fn get_type_object(&self) -> *mut ffi::PyTypeObject; -} - -/// Hold the GIL and only obtain/construct type objects lazily when required. +/// The different solutions to this have different trade-offs. #[derive(Clone, Copy)] -pub struct LazyTypeProvider<'py>(PhantomData<&'py ()>); -impl<'py> LazyTypeProvider<'py> { - pub fn new(_py: Python<'py>) -> Self { - Self(PhantomData) - } +pub enum TypeObjectStrategy<'a> { + Lazy(Python<'a>), + AssumeInit(PhantomData<&'a ()>), } -impl<'py> TypeObjectProvider for LazyTypeProvider<'py> { - fn get_type_object(&self) -> *mut ffi::PyTypeObject { - let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - T::type_object_raw(py) + +impl<'a> TypeObjectStrategy<'a> { + /// Hold the GIL and only obtain/construct type objects lazily when required. + pub fn lazy(py: Python<'a>) -> Self { + TypeObjectStrategy::Lazy(py) } -} -/// Assume that `PyTypeInfo::type_object_raw()` has been called for any of the required type objects. -/// -/// once initialized, the type objects are cached and can be obtained without holding the GIL. -#[derive(Clone, Copy)] -pub struct AssumeInitializedTypeProvider; -impl AssumeInitializedTypeProvider { - /// Create a type provider that assumes that whatever `T` it is called with has already been initialized. + /// Assume that `PyTypeInfo::type_object_raw()` has been called for any of the required type objects. + /// + /// Once initialized, the type objects are cached and can be obtained without holding the GIL. /// /// # Safety /// - /// - ensure that any `T` that may be used with this object has already been initialized + /// - Ensure that any `T` that may be used with this strategy has already been initialized /// by calling `T::type_object_raw()`. - /// - only `PyTypeInfo::OPAQUE` classes require type objects for traversal so if this object is only + /// - Only `PyTypeInfo::OPAQUE` classes require type objects for traversal so if this strategy is only /// used with non-opaque classes then no action is required. - /// - when used with `PyClassRecursiveOperations` or `GetBorrowChecker`, the object may be used with + /// - When used with `PyClassRecursiveOperations` or `GetBorrowChecker`, the strategy may be used with /// base classes as well as the most derived type. /// `PyClassRecursiveOperations::ensure_type_objects_initialized()` can be used to initialize /// all base classes above the given type. - pub unsafe fn new() -> Self { - Self - } -} -impl TypeObjectProvider for AssumeInitializedTypeProvider { - fn get_type_object(&self) -> *mut ffi::PyTypeObject { - T::try_get_type_object_raw().unwrap_or_else(|| { - panic!( - "type object for {} not initialized", - std::any::type_name::() - ) - }) + pub unsafe fn assume_init() -> Self { + TypeObjectStrategy::AssumeInit(PhantomData) } } @@ -337,13 +324,13 @@ impl PyObjectLayout { /// Obtain a pointer to the contents of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_contents_ptr( + pub(crate) unsafe fn get_contents_ptr( obj: *mut ffi::PyObject, - type_provider: P, + strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { debug_assert!(!obj.is_null()); if ::OPAQUE { - opaque_layout::get_contents_ptr(obj, type_provider) + opaque_layout::get_contents_ptr(obj, strategy) } else { let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); // indicates `ob_base` has type InvalidBaseLayout @@ -356,12 +343,12 @@ impl PyObjectLayout { } } - pub(crate) fn get_contents( - obj: &ffi::PyObject, - type_provider: P, - ) -> &PyClassObjectContents { + pub(crate) fn get_contents<'a, T: PyClassImpl + PyTypeInfo>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a PyClassObjectContents { unsafe { - &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) + &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut(), strategy) .cast_const() } } @@ -369,35 +356,33 @@ impl PyObjectLayout { /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. /// /// Safety: the provided object must be valid and have the layout indicated by `T` - pub(crate) unsafe fn get_data_ptr( + pub(crate) unsafe fn get_data_ptr( obj: *mut ffi::PyObject, - type_provider: P, + strategy: TypeObjectStrategy<'_>, ) -> *mut T { - let contents = PyObjectLayout::get_contents_ptr::(obj, type_provider); + let contents = PyObjectLayout::get_contents_ptr::(obj, strategy); (*contents).value.get() } - pub(crate) fn get_data( - obj: &ffi::PyObject, - type_provider: P, - ) -> &T { - unsafe { - &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), type_provider) - } + pub(crate) fn get_data<'a, T: PyClassImpl + PyTypeInfo>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a T { + unsafe { &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), strategy) } } - pub(crate) fn get_borrow_checker<'o, T: PyClassImpl + PyTypeInfo>( + pub(crate) fn get_borrow_checker<'a, T: PyClassImpl + PyTypeInfo>( py: Python<'_>, - obj: &'o ffi::PyObject, - ) -> &'o ::Checker { - T::PyClassMutability::borrow_checker(obj, LazyTypeProvider::new(py)) + obj: &'a ffi::PyObject, + ) -> &'a ::Checker { + T::PyClassMutability::borrow_checker(obj, TypeObjectStrategy::lazy(py)) } pub(crate) fn ensure_threadsafe( py: Python<'_>, obj: &ffi::PyObject, ) { - PyClassRecursiveOperations::::ensure_threadsafe(obj, LazyTypeProvider::new(py)); + PyClassRecursiveOperations::::ensure_threadsafe(obj, TypeObjectStrategy::lazy(py)); } /// Clean up then free the memory associated with `obj`. diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 91d5b5ab471..d936c995ac0 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -3,7 +3,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::PyClassBaseType; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; -use crate::pycell::layout::{LazyTypeProvider, PyObjectLayout}; +use crate::pycell::layout::{PyObjectLayout, TypeObjectStrategy}; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; use crate::{ffi::PyTypeObject, pycell::layout::PyClassObjectContents}; @@ -167,7 +167,7 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; std::ptr::write( - PyObjectLayout::get_contents_ptr::(obj, LazyTypeProvider::new(py)), + PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)), PyClassObjectContents::new(init), ); From 69329ee225a463f6e025e30d7616fd3c8b76cbc5 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 11 Nov 2024 20:47:05 +0000 Subject: [PATCH 36/41] add pyclass option to force classes to use the opaque layout --- pyo3-macros-backend/src/attributes.rs | 3 ++- pyo3-macros-backend/src/pyclass.rs | 13 ++++++++++++- src/pycell/layout.rs | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index 94526e7dafc..e3663f4d5a0 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -20,16 +20,17 @@ pub mod kw { syn::custom_keyword!(eq_int); syn::custom_keyword!(extends); syn::custom_keyword!(freelist); + syn::custom_keyword!(from_item_all); syn::custom_keyword!(from_py_with); syn::custom_keyword!(frozen); syn::custom_keyword!(get); syn::custom_keyword!(get_all); syn::custom_keyword!(hash); syn::custom_keyword!(item); - syn::custom_keyword!(from_item_all); syn::custom_keyword!(mapping); syn::custom_keyword!(module); syn::custom_keyword!(name); + syn::custom_keyword!(opaque); syn::custom_keyword!(ord); syn::custom_keyword!(pass_module); syn::custom_keyword!(rename_all); diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 98ca9b653b9..541574596c1 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -72,6 +72,7 @@ pub struct PyClassPyO3Options { pub module: Option, pub name: Option, pub ord: Option, + pub opaque: Option, pub rename_all: Option, pub sequence: Option, pub set_all: Option, @@ -95,6 +96,7 @@ pub enum PyClassPyO3Option { Module(ModuleAttribute), Name(NameAttribute), Ord(kw::ord), + Opaque(kw::opaque), RenameAll(RenameAllAttribute), Sequence(kw::sequence), SetAll(kw::set_all), @@ -133,6 +135,8 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Name) } else if lookahead.peek(attributes::kw::ord) { input.parse().map(PyClassPyO3Option::Ord) + } else if lookahead.peek(attributes::kw::opaque) { + input.parse().map(PyClassPyO3Option::Opaque) } else if lookahead.peek(kw::rename_all) { input.parse().map(PyClassPyO3Option::RenameAll) } else if lookahead.peek(attributes::kw::sequence) { @@ -205,6 +209,7 @@ impl PyClassPyO3Options { PyClassPyO3Option::Module(module) => set_option!(module), PyClassPyO3Option::Name(name) => set_option!(name), PyClassPyO3Option::Ord(ord) => set_option!(ord), + PyClassPyO3Option::Opaque(opaque) => set_option!(opaque), PyClassPyO3Option::RenameAll(rename_all) => set_option!(rename_all), PyClassPyO3Option::Sequence(sequence) => set_option!(sequence), PyClassPyO3Option::SetAll(set_all) => set_option!(set_all), @@ -1807,11 +1812,17 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre quote! { ::core::option::Option::None } }; + let opaque = if attr.options.opaque.is_some() { + quote! { true } + } else { + quote! { <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE } + }; + quote! { unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; - const OPAQUE: bool = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE; + const OPAQUE: bool = #opaque; #[inline] fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 206edc7b7d1..2095c23cb3a 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -329,7 +329,7 @@ impl PyObjectLayout { strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { debug_assert!(!obj.is_null()); - if ::OPAQUE { + if T::OPAQUE { opaque_layout::get_contents_ptr(obj, strategy) } else { let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); From d9d9fdc4265152a273da14e13d4893470b5dbf3e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 22 Nov 2024 21:47:18 +0000 Subject: [PATCH 37/41] fix remaining issues and add tests --- Contributing.md | 3 +- guide/pyclass-parameters.md | 1 + pyo3-macros-backend/src/pyclass.rs | 16 +- src/impl_/pyclass.rs | 12 +- src/impl_/pyclass_init.rs | 7 +- src/impl_/pymethods.rs | 16 +- src/instance.rs | 44 +- src/internal/get_slot.rs | 1 + src/pycell.rs | 26 +- src/pycell/borrow_checker.rs | 6 +- src/pycell/layout.rs | 1374 +++++++++++++++-- src/type_object.rs | 8 +- tests/test_class_basics.rs | 6 +- tests/test_class_init.rs | 1 + tests/test_compile_error.rs | 3 +- tests/test_inheritance.rs | 52 +- tests/ui/invalid_extend_variable_sized.rs | 14 - tests/ui/invalid_extend_variable_sized.stderr | 15 - tests/ui/invalid_frozen_pyclass_borrow.rs | 4 +- tests/ui/invalid_frozen_pyclass_borrow.stderr | 20 +- tests/ui/invalid_opaque.rs | 6 + tests/ui/invalid_opaque.stderr | 16 + tests/ui/invalid_pyclass_args.stderr | 4 +- 23 files changed, 1425 insertions(+), 230 deletions(-) delete mode 100644 tests/ui/invalid_extend_variable_sized.rs delete mode 100644 tests/ui/invalid_extend_variable_sized.stderr create mode 100644 tests/ui/invalid_opaque.rs create mode 100644 tests/ui/invalid_opaque.stderr diff --git a/Contributing.md b/Contributing.md index 7ab6cfe27ea..2615028a1ba 100644 --- a/Contributing.md +++ b/Contributing.md @@ -122,7 +122,8 @@ The easiest way to configure the python version is to install with the system pa `Py_LIMITED_API` can be controlled with the `abi3` feature of the `pyo3` crate: ``` -PYO3_PYTHON=/path/to/python cargo nextest run --package pyo3 --features abi3 ... +LD_LIBRARY_PATH=/lib PYO3_PYTHON=/bin/python \ + cargo nextest run --package pyo3 --features abi3 ... ``` use the `PYO3_PRINT_CONFIG=1` to check the identified configuration. diff --git a/guide/pyclass-parameters.md b/guide/pyclass-parameters.md index b471f5dd3ae..1ecf7fa25cc 100644 --- a/guide/pyclass-parameters.md +++ b/guide/pyclass-parameters.md @@ -24,6 +24,7 @@ | `text_signature = "(arg1, arg2, ...)"` | Sets the text signature for the Python class' `__new__` method. | | `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. | | `weakref` | Allows this class to be [weakly referenceable][params-6]. | +| `opaque` | Forces the use of the 'opaque layout' ([PEP 697](https://peps.python.org/pep-0697/)) for this class and any subclasses that extend it. Primarily used for internal testing. | All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or more accompanying `#[pyo3(...)]` annotations, e.g.: diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 541574596c1..5ec53598bde 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1196,6 +1196,7 @@ fn impl_complex_enum_variant_match_args( quote! { &'static str } }); parse_quote! { + #[allow(non_upper_case_globals)] const __match_args__: ( #(#args_tp,)* ) = ( #(stringify!(#field_names),)* ); @@ -1813,16 +1814,25 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre }; let opaque = if attr.options.opaque.is_some() { - quote! { true } + quote! { + #[cfg(Py_3_12)] + const OPAQUE: bool = true; + + #[cfg(not(Py_3_12))] + ::core::compile_error!("#[pyclass(opaque)] requires python 3.12 or later"); + } } else { - quote! { <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE } + // if opaque is not supported an error will be raised at construction + quote! { + const OPAQUE: bool = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE; + } }; quote! { unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; - const OPAQUE: bool = #opaque; + #opaque #[inline] fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index a0e6c22adb2..68bd74bfc1e 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1,3 +1,5 @@ +#[cfg(Py_3_12)] +use crate::pycell::layout::TypeObjectStrategy; use crate::{ conversion::IntoPyObject, exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError}, @@ -9,7 +11,7 @@ use crate::{ pymethods::{PyGetterDef, PyMethodDefType}, }, pycell::{ - layout::{PyObjectLayout, PyObjectRecursiveOperations, TypeObjectStrategy}, + layout::{usize_to_py_ssize, PyObjectLayout, PyObjectRecursiveOperations}, PyBorrowError, }, type_object::PyLayout, @@ -1188,7 +1190,7 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( } /// Offset of a field within a PyObject in bytes. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PyObjectOffset { /// An offset relative to the start of the object Absolute(ffi::Py_ssize_t), @@ -1202,10 +1204,7 @@ impl std::ops::Add for PyObjectOffset { type Output = PyObjectOffset; fn add(self, rhs: usize) -> Self::Output { - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - let rhs: ffi::Py_ssize_t = rhs.try_into().expect("offset should fit in Py_ssize_t"); - + let rhs = usize_to_py_ssize(rhs); match self { PyObjectOffset::Absolute(offset) => PyObjectOffset::Absolute(offset + rhs), PyObjectOffset::Relative(offset) => PyObjectOffset::Relative(offset + rhs), @@ -1553,6 +1552,7 @@ where } #[cfg(not(Py_3_12))] PyObjectOffset::Relative(_) => { + let _ = py; panic!("relative offsets not valid before python 3.12"); } }; diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 91359cdb04b..caf9e8d9619 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -5,13 +5,18 @@ use crate::pycell::layout::{PyClassObjectContents, PyObjectLayout, TypeObjectStr use crate::types::PyType; use crate::{ffi, Borrowed, PyClass, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; +use std::any::TypeId; use std::marker::PhantomData; pub unsafe fn initialize_with_default( py: Python<'_>, obj: *mut ffi::PyObject, ) { - // TODO(matt): need to initialize all base classes, should be recursive + if TypeId::of::() != TypeId::of::() { + // only sets the PyClassContents of the 'most derived type' + // so any parent pyclasses would remain uninitialized. + panic!("initialize_with_default does not currently support multi-level inheritance"); + } std::ptr::write( PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)), PyClassObjectContents::new(T::default()), diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 88ba51c1fdb..44f26c4d252 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -286,16 +286,6 @@ pub unsafe fn _call_traverse( where T: PyClass, { - { - let py = Python::assume_gil_acquired(); - // allows functions that traverse the contents of `slf` below to assume that all - // the type objects are initialized. Only classes using the opaque layout use - // type objects to traverse the object. - if T::OPAQUE { - PyClassRecursiveOperations::::ensure_type_objects_initialized(py); - } - } - // It is important the implementation of `__traverse__` cannot safely access the GIL, // c.f. https://github.com/PyO3/pyo3/issues/3165, and hence we do not expose our GIL // token to the user code and lock safe methods for acquiring the GIL. @@ -315,8 +305,9 @@ where // traversal is running so no mutations can occur. let raw_obj = &*slf; - // SAFETY: type objects for `T` and all base classes of `T` have been initialized - // above if they are required. + // SAFETY: type objects for `T` and all ancestors of `T` are created the first time an + // instance of `T` is created. Since `slf` is an instance of `T` the type objects must + // have been created. let strategy = TypeObjectStrategy::assume_init(); let retval = @@ -339,7 +330,6 @@ where // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); - // Safety: type object is manually initialized above let instance = PyObjectLayout::get_data::(raw_obj, strategy); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/instance.rs b/src/instance.rs index fd4119452d3..8ecc84973ab 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -326,25 +326,10 @@ where PyRefMut::try_borrow(self) } - /// Call this function before using `get()` - pub fn enable_get(py: Python<'_>) - where - T: PyTypeInfo, - { - // only classes using the opaque layout require type objects for traversal - if T::OPAQUE { - let _ = T::type_object_raw(py); - } - } - /// Provide an immutable borrow of the value `T` without acquiring the GIL. /// /// This is available if the class is [`frozen`][macro@crate::pyclass] and [`Sync`]. /// - /// # Safety - /// - /// `enable_get()` must have been called for `T` beforehand. - /// /// # Examples /// /// ``` @@ -365,11 +350,10 @@ where /// }); /// ``` #[inline] - pub unsafe fn get(&self) -> &T + pub fn get(&self) -> &T where T: PyClass + Sync, { - // Safety: `enable_get()` has already been called for `T`. self.1.get() } @@ -1298,24 +1282,10 @@ where self.bind(py).try_borrow_mut() } - /// Call this function before using `get()` - pub fn enable_get(py: Python<'_>) - where - T: PyTypeInfo, - { - // only classes using the opaque layout require type objects for traversal - if T::OPAQUE { - let _ = T::type_object_raw(py); - } - } - /// Provide an immutable borrow of the value `T` without acquiring the GIL. /// /// This is available if the class is [`frozen`][macro@crate::pyclass] and [`Sync`]. /// - /// # Safety - /// - /// `enable_get()` must have been called for `T` beforehand. /// /// # Examples /// @@ -1338,13 +1308,14 @@ where /// # Python::with_gil(move |_py| drop(cell)); /// ``` #[inline] - pub unsafe fn get(&self) -> &T + pub fn get(&self) -> &T where T: PyClass + Sync, { - // Safety: `enable_get()` has already been called for `T`. + // Safety: the PyTypeObject for T will have been created when the first instance of T was created. + // Since Py contains an instance of T the type object must have already been created. let strategy = unsafe { TypeObjectStrategy::assume_init() }; - PyObjectLayout::get_data::(self.as_raw_ref(), strategy) + unsafe { PyObjectLayout::get_data::(self.as_raw_ref(), strategy) } } } @@ -2380,11 +2351,10 @@ a = A() #[test] fn test_frozen_get() { Python::with_gil(|py| { - Py::::enable_get(py); for i in 0..10 { let instance = Py::new(py, FrozenClass(i)).unwrap(); - assert_eq!(unsafe { instance.get().0 }, i); - assert_eq!(unsafe { instance.bind(py).get().0 }, i); + assert_eq!(instance.get().0, i); + assert_eq!(instance.bind(py).get().0, i); } }) } diff --git a/src/internal/get_slot.rs b/src/internal/get_slot.rs index 260893d4204..55a4bb240fe 100644 --- a/src/internal/get_slot.rs +++ b/src/internal/get_slot.rs @@ -126,6 +126,7 @@ impl_slots! { TP_BASE: (Py_tp_base, tp_base) -> *mut ffi::PyTypeObject, TP_CLEAR: (Py_tp_clear, tp_clear) -> Option, TP_DESCR_GET: (Py_tp_descr_get, tp_descr_get) -> Option, + TP_DEALLOC: (Py_tp_dealloc, tp_dealloc) -> Option, TP_FREE: (Py_tp_free, tp_free) -> Option, TP_TRAVERSE: (Py_tp_traverse, tp_traverse) -> Option, } diff --git a/src/pycell.rs b/src/pycell.rs index 4a066d78ee5..b2b4a8f838c 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -312,8 +312,8 @@ impl<'py, T: PyClass> PyRef<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let raw_obj = obj.as_raw_ref(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - PyObjectLayout::ensure_threadsafe::(py, raw_obj); - let borrow_checker = PyObjectLayout::get_borrow_checker::(py, raw_obj); + unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; borrow_checker .try_borrow() .map(|_| Self { inner: obj.clone() }) @@ -441,7 +441,8 @@ impl<'py, T: PyClass> Deref for PyRef<'py, T> { #[inline] fn deref(&self) -> &T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - PyObjectLayout::get_data::(self.inner.as_raw_ref(), TypeObjectStrategy::lazy(py)) + let obj = self.inner.as_raw_ref(); + unsafe { PyObjectLayout::get_data::(obj, TypeObjectStrategy::lazy(py)) } } } @@ -449,7 +450,7 @@ impl<'py, T: PyClass> Drop for PyRef<'py, T> { fn drop(&mut self) { let obj = self.inner.as_raw_ref(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - let borrow_checker = PyObjectLayout::get_borrow_checker::(py, obj); + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; borrow_checker.release_borrow(); } } @@ -571,8 +572,8 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { let raw_obj = obj.as_raw_ref(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - PyObjectLayout::ensure_threadsafe::(py, raw_obj); - let borrow_checker = PyObjectLayout::get_borrow_checker::(py, raw_obj); + unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; borrow_checker .try_borrow_mut() .map(|_| Self { inner: obj.clone() }) @@ -628,7 +629,8 @@ impl<'py, T: PyClass> Deref for PyRefMut<'py, T> { #[inline] fn deref(&self) -> &T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - PyObjectLayout::get_data::(self.inner.as_raw_ref(), TypeObjectStrategy::lazy(py)) + let obj = self.inner.as_raw_ref(); + unsafe { PyObjectLayout::get_data::(obj, TypeObjectStrategy::lazy(py)) } } } @@ -636,12 +638,8 @@ impl<'py, T: PyClass> DerefMut for PyRefMut<'py, T> { #[inline] fn deref_mut(&mut self) -> &mut T { let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - unsafe { - &mut *PyObjectLayout::get_data_ptr::( - self.inner.as_ptr(), - TypeObjectStrategy::lazy(py), - ) - } + let obj = self.inner.as_ptr(); + unsafe { &mut *PyObjectLayout::get_data_ptr::(obj, TypeObjectStrategy::lazy(py)) } } } @@ -649,7 +647,7 @@ impl<'py, T: PyClass> Drop for PyRefMut<'py, T> { fn drop(&mut self) { let obj = self.inner.get_raw_object(); let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; - let borrow_checker = PyObjectLayout::get_borrow_checker::(py, obj); + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; borrow_checker.release_borrow_mut(); } } diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index f63d960116b..2673c9d3a32 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -163,7 +163,7 @@ impl PyClassBorrowChecker for BorrowChecker { } fn release_borrow_mut(&self) { - self.0 .0.store(BorrowFlag::UNUSED, Ordering::Release) + self.0.0.store(BorrowFlag::UNUSED, Ordering::Release) } } @@ -179,7 +179,7 @@ impl + PyTypeInfo> GetBorrowChecker obj: &'a ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> &'a BorrowChecker { - let contents = PyObjectLayout::get_contents::(obj, strategy); + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; &contents.borrow_checker } } @@ -189,7 +189,7 @@ impl + PyTypeInfo> GetBorrowChecker obj: &'a ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> &'a EmptySlot { - let contents = PyObjectLayout::get_contents::(obj, strategy); + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; &contents.borrow_checker } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 2095c23cb3a..2acf899feff 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -9,7 +9,7 @@ use std::ptr::addr_of_mut; use crate::impl_::pyclass::{ PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, }; -use crate::internal::get_slot::TP_FREE; +use crate::internal::get_slot::{TP_DEALLOC, TP_FREE}; use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; use crate::type_object::PyNativeType; use crate::types::PyType; @@ -21,10 +21,10 @@ use crate::types::PyTypeMethods; use super::borrow_checker::PyClassMutability; use super::{ptr_from_ref, PyBorrowError}; -/// The data of a `ffi::PyObject` specifically relating to type `T`. +/// The data of a [ffi::PyObject] specifically relating to type `T`. /// /// In an inheritance hierarchy where `#[pyclass(extends=PyDict)] struct A;` and `#[pyclass(extends=A)] struct B;` -/// a `ffi::PyObject` of type `B` has separate memory for `ffi::PyDictObject` (the base native type) and +/// a [ffi::PyObject] of type `B` has separate memory for [ffi::PyDictObject] (the base native type) and /// `PyClassObjectContents` and `PyClassObjectContents`. The memory associated with `A` or `B` can be obtained /// using `PyObjectLayout::get_contents::()` (where `T=A` or `T=B`). #[repr(C)] @@ -33,9 +33,9 @@ pub(crate) struct PyClassObjectContents { pub(crate) value: ManuallyDrop>, pub(crate) borrow_checker: ::Storage, pub(crate) thread_checker: T::ThreadChecker, - /// A pointer to a `PyObject` if `T` is annotated with `#[pyclass(dict)]` and a zero-sized field otherwise. + /// A pointer to a [ffi::PyObject]` if `T` is annotated with `#[pyclass(dict)]` and a zero-sized field otherwise. pub(crate) dict: T::Dict, - /// A pointer to a `PyObject` if `T` is annotated with `#[pyclass(weakref)]` and a zero-sized field otherwise. + /// A pointer to a [ffi::PyObject] if `T` is annotated with `#[pyclass(weakref)]` and a zero-sized field otherwise. pub(crate) weakref: T::WeakRef, } @@ -67,14 +67,14 @@ impl PyClassObjectContents { /// then calling a method on a PyObject of type `B` will call the method for `B`, then `A`, then `PyDict`. #[doc(hidden)] pub trait PyObjectRecursiveOperations { - /// `PyTypeInfo::type_object_raw()` may create type objects lazily. + /// [PyTypeInfo::type_object_raw()] may create type objects lazily. /// This method ensures that the type objects for all ancestor types of the provided object. - unsafe fn ensure_type_objects_initialized(py: Python<'_>); + fn ensure_type_objects_initialized(py: Python<'_>); - /// Call `PyClassThreadChecker::ensure` on all ancestor types of the provided object. + /// Call [PyClassThreadChecker::ensure()] on all ancestor types of the provided object. fn ensure_threadsafe(obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>); - /// Call `PyClassThreadChecker::check` on all ancestor types of the provided object. + /// Call [PyClassThreadChecker::check()] on all ancestor types of the provided object. fn check_threadsafe( obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>, @@ -88,17 +88,17 @@ pub trait PyObjectRecursiveOperations { unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject); } -/// Used to fill out `PyClassBaseType::RecursiveOperations` for instances of `PyClass` +/// Used to fill out [PyClassBaseType::RecursiveOperations] for instances of `PyClass` pub struct PyClassRecursiveOperations(PhantomData); impl PyObjectRecursiveOperations for PyClassRecursiveOperations { - unsafe fn ensure_type_objects_initialized(py: Python<'_>) { + fn ensure_type_objects_initialized(py: Python<'_>) { let _ = ::type_object_raw(py); ::RecursiveOperations::ensure_type_objects_initialized(py); } fn ensure_threadsafe(obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>) { - let contents = PyObjectLayout::get_contents::(obj, strategy); + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; contents.thread_checker.ensure(); ::RecursiveOperations::ensure_threadsafe(obj, strategy); } @@ -107,7 +107,7 @@ impl PyObjectRecursiveOperations for PyClassRecursi obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> Result<(), PyBorrowError> { - let contents = PyObjectLayout::get_contents::(obj, strategy); + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; if !contents.thread_checker.check() { return Err(PyBorrowError { _private: () }); } @@ -123,13 +123,13 @@ impl PyObjectRecursiveOperations for PyClassRecursi } } -/// Used to fill out `PyClassBaseType::RecursiveOperations` for native types +/// Used to fill out [PyClassBaseType::RecursiveOperations] for native types pub struct PyNativeTypeRecursiveOperations(PhantomData); impl PyObjectRecursiveOperations for PyNativeTypeRecursiveOperations { - unsafe fn ensure_type_objects_initialized(py: Python<'_>) { + fn ensure_type_objects_initialized(py: Python<'_>) { let _ = ::type_object_raw(py); } @@ -162,17 +162,23 @@ impl PyObjectRecursiveOperations // the 'most derived class' of `obj`. i.e. the result of calling `type(obj)`. let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); - // TODO(matt): is this correct? - // For `#[pyclass]` types which inherit from PyAny or PyType, we can just call tp_free - let is_base_object = type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); - let is_metaclass = type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type); - if is_base_object || is_metaclass { + if type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { + // the `PyBaseObject_Type` destructor (tp_dealloc) just calls tp_free so we can do this directly let tp_free = actual_type .get_slot(TP_FREE) .expect("base type should have tp_free"); return tp_free(obj.cast()); } + if type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type) { + let tp_dealloc = PyType::from_borrowed_type_ptr(py, type_ptr) + .get_slot(TP_DEALLOC) + .expect("PyType_Type should have tp_dealloc"); + // `PyType_Type::dealloc` calls `Py_GC_UNTRACK` so we have to re-track before deallocating + ffi::PyObject_GC_Track(obj.cast()); + return tp_dealloc(obj.cast()); + } + // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. #[cfg(not(Py_LIMITED_API))] { @@ -201,42 +207,47 @@ impl PyObjectRecursiveOperations /// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). #[doc(hidden)] pub(crate) mod opaque_layout { - use super::PyClassObjectContents; - use super::TypeObjectStrategy; - use crate::{ffi, impl_::pyclass::PyClassImpl, PyTypeInfo}; + #[cfg(Py_3_12)] + use super::{PyClassObjectContents, TypeObjectStrategy}; + #[cfg(Py_3_12)] + use crate::ffi; + use crate::{impl_::pyclass::PyClassImpl, PyTypeInfo}; #[cfg(Py_3_12)] pub fn get_contents_ptr( obj: *mut ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { - #[cfg(Py_3_12)] - { - let type_obj = match strategy { - TypeObjectStrategy::Lazy(py) => T::type_object_raw(py), - TypeObjectStrategy::AssumeInit(_) => { - T::try_get_type_object_raw().unwrap_or_else(|| { - panic!( - "type object for {} not initialized", - std::any::type_name::() - ) - }) - } - }; - assert!(!type_obj.is_null(), "type object is NULL"); - let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; - assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); - pointer.cast() - } - - #[cfg(not(Py_3_12))] - panic_unsupported(); + let type_obj = match strategy { + TypeObjectStrategy::Lazy(py) => T::type_object_raw(py), + TypeObjectStrategy::AssumeInit(_) => { + T::try_get_type_object_raw().unwrap_or_else(|| { + panic!( + "type object for {} not initialized", + std::any::type_name::() + ) + }) + } + }; + assert!(!type_obj.is_null(), "type object is NULL"); + debug_assert!( + unsafe { ffi::PyType_IsSubtype(ffi::Py_TYPE(obj), type_obj) } == 1, + "the object is not an instance of {}", + std::any::type_name::() + ); + let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; + assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); + pointer.cast() } #[inline(always)] #[cfg(not(Py_3_12))] - fn panic_unsupported() { - panic!("opaque layout not supported until python 3.12"); + pub fn panic_unsupported() -> ! { + assert!(T::OPAQUE); + panic!( + "The opaque object layout (used by {}) is not supported until python 3.12", + std::any::type_name::() + ); } } @@ -280,11 +291,11 @@ pub(crate) mod static_layout { unsafe impl PyLayout for InvalidStaticLayout {} } -/// The method to use for obtaining a `*mut ffi::PyTypeObject` pointer describing `T: PyTypeInfo` for -/// use with `PyObjectLayout` functions. +/// The method to use for obtaining a [ffi::PyTypeObject] pointer describing `T: PyTypeInfo` for +/// use with [PyObjectLayout] functions. /// -/// `PyTypeInfo::type_object_raw()` requires the GIL to be held because it may lazily construct the type object. -/// Some situations require that the GIL is not held so `PyObjectLayout` cannot call this method directly. +/// [PyTypeInfo::type_object_raw()] requires the GIL to be held because it may lazily construct the type object. +/// Some situations require that the GIL is not held so [PyObjectLayout] cannot call this method directly. /// The different solutions to this have different trade-offs. #[derive(Clone, Copy)] pub enum TypeObjectStrategy<'a> { @@ -298,39 +309,49 @@ impl<'a> TypeObjectStrategy<'a> { TypeObjectStrategy::Lazy(py) } - /// Assume that `PyTypeInfo::type_object_raw()` has been called for any of the required type objects. + /// Assume that [PyTypeInfo::type_object_raw()] has been called for any of the required type objects. /// /// Once initialized, the type objects are cached and can be obtained without holding the GIL. /// /// # Safety /// /// - Ensure that any `T` that may be used with this strategy has already been initialized - /// by calling `T::type_object_raw()`. - /// - Only `PyTypeInfo::OPAQUE` classes require type objects for traversal so if this strategy is only + /// by calling [PyTypeInfo::type_object_raw()]. + /// - Only [PyTypeInfo::OPAQUE] classes require type objects for traversal so if this strategy is only /// used with non-opaque classes then no action is required. - /// - When used with `PyClassRecursiveOperations` or `GetBorrowChecker`, the strategy may be used with + /// - When used with [PyClassRecursiveOperations] or [GetBorrowChecker], the strategy may be used with /// base classes as well as the most derived type. - /// `PyClassRecursiveOperations::ensure_type_objects_initialized()` can be used to initialize + /// [PyClassRecursiveOperations::ensure_type_objects_initialized()] can be used to initialize /// all base classes above the given type. pub unsafe fn assume_init() -> Self { TypeObjectStrategy::AssumeInit(PhantomData) } } -/// Functions for working with `PyObject`s +/// Functions for working with [ffi::PyObject]s pub(crate) struct PyObjectLayout {} impl PyObjectLayout { - /// Obtain a pointer to the contents of a `PyObject` of type `T`. + /// Obtain a pointer to the portion of `obj` relating to the type `T` /// - /// Safety: the provided object must be valid and have the layout indicated by `T` + /// # Safety + /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. pub(crate) unsafe fn get_contents_ptr( obj: *mut ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { - debug_assert!(!obj.is_null()); + debug_assert!(!obj.is_null(), "get_contents_ptr of null object"); if T::OPAQUE { - opaque_layout::get_contents_ptr(obj, strategy) + #[cfg(Py_3_12)] + { + opaque_layout::get_contents_ptr(obj, strategy) + } + + #[cfg(not(Py_3_12))] + { + let _ = strategy; + opaque_layout::panic_unsupported::(); + } } else { let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); // indicates `ob_base` has type InvalidBaseLayout @@ -343,7 +364,11 @@ impl PyObjectLayout { } } - pub(crate) fn get_contents<'a, T: PyClassImpl + PyTypeInfo>( + /// Obtain a reference to the portion of `obj` relating to the type `T` + /// + /// # Safety + /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_contents<'a, T: PyClassImpl + PyTypeInfo>( obj: &'a ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> &'a PyClassObjectContents { @@ -353,9 +378,10 @@ impl PyObjectLayout { } } - /// obtain a pointer to the pyclass struct of a `PyObject` of type `T`. + /// Obtain a pointer to the portion of `obj` containing the data for `T` /// - /// Safety: the provided object must be valid and have the layout indicated by `T` + /// # Safety + /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. pub(crate) unsafe fn get_data_ptr( obj: *mut ffi::PyObject, strategy: TypeObjectStrategy<'_>, @@ -364,21 +390,37 @@ impl PyObjectLayout { (*contents).value.get() } - pub(crate) fn get_data<'a, T: PyClassImpl + PyTypeInfo>( + /// Obtain a reference to the portion of `obj` containing the data for `T` + /// + /// # Safety + /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_data<'a, T: PyClassImpl + PyTypeInfo>( obj: &'a ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> &'a T { unsafe { &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), strategy) } } - pub(crate) fn get_borrow_checker<'a, T: PyClassImpl + PyTypeInfo>( + /// Obtain a reference to the borrow checker for `obj` + /// + /// Note: this method is for convenience. The implementation is in [GetBorrowChecker]. + /// + /// # Safety + /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_borrow_checker<'a, T: PyClassImpl + PyTypeInfo>( py: Python<'_>, obj: &'a ffi::PyObject, ) -> &'a ::Checker { T::PyClassMutability::borrow_checker(obj, TypeObjectStrategy::lazy(py)) } - pub(crate) fn ensure_threadsafe( + /// Ensure that `obj` is thread safe. + /// + /// Note: this method is for convenience. The implementation is in [PyClassRecursiveOperations]. + /// + /// # Safety + /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn ensure_threadsafe( py: Python<'_>, obj: &ffi::PyObject, ) { @@ -387,8 +429,13 @@ impl PyObjectLayout { /// Clean up then free the memory associated with `obj`. /// + /// Note: this method is for convenience. The implementation is in [PyClassRecursiveOperations]. + /// /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - pub(crate) fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + pub(crate) unsafe fn deallocate( + py: Python<'_>, + obj: *mut ffi::PyObject, + ) { unsafe { PyClassRecursiveOperations::::deallocate(py, obj); }; @@ -399,7 +446,7 @@ impl PyObjectLayout { /// Use instead of `deallocate()` if `T` has the `Py_TPFLAGS_HAVE_GC` flag set. /// /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) - pub(crate) fn deallocate_with_gc( + pub(crate) unsafe fn deallocate_with_gc( py: Python<'_>, obj: *mut ffi::PyObject, ) { @@ -414,32 +461,32 @@ impl PyObjectLayout { /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) - pub(crate) fn basicsize() -> ffi::Py_ssize_t { - if ::OPAQUE { + pub(crate) fn basicsize() -> ffi::Py_ssize_t { + if T::OPAQUE { #[cfg(Py_3_12)] { // negative to indicate 'extra' space that python will allocate - // specifically for `T` excluding the base class + // specifically for `T` excluding the base class. -usize_to_py_ssize(std::mem::size_of::>()) } #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); + opaque_layout::panic_unsupported::(); } else { usize_to_py_ssize(std::mem::size_of::>()) } } /// Gets the offset of the contents from the start of the struct in bytes. - pub(crate) fn contents_offset() -> PyObjectOffset { - if ::OPAQUE { + pub(crate) fn contents_offset() -> PyObjectOffset { + if T::OPAQUE { #[cfg(Py_3_12)] { PyObjectOffset::Relative(0) } #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); + opaque_layout::panic_unsupported::(); } else { PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( static_layout::PyStaticClassLayout, @@ -449,8 +496,8 @@ impl PyObjectLayout { } /// Gets the offset of the dictionary from the start of the struct in bytes. - pub(crate) fn dict_offset() -> PyObjectOffset { - if ::OPAQUE { + pub(crate) fn dict_offset() -> PyObjectOffset { + if T::OPAQUE { #[cfg(Py_3_12)] { PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( @@ -460,7 +507,7 @@ impl PyObjectLayout { } #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); + opaque_layout::panic_unsupported::(); } else { let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + memoffset::offset_of!(PyClassObjectContents, dict); @@ -470,8 +517,8 @@ impl PyObjectLayout { } /// Gets the offset of the weakref list from the start of the struct in bytes. - pub(crate) fn weaklist_offset() -> PyObjectOffset { - if ::OPAQUE { + pub(crate) fn weaklist_offset() -> PyObjectOffset { + if T::OPAQUE { #[cfg(Py_3_12)] { PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( @@ -481,7 +528,7 @@ impl PyObjectLayout { } #[cfg(not(Py_3_12))] - opaque_layout::panic_unsupported(); + opaque_layout::panic_unsupported::(); } else { let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + memoffset::offset_of!(PyClassObjectContents, weakref); @@ -492,35 +539,497 @@ impl PyObjectLayout { } /// Py_ssize_t may not be equal to isize on all platforms -fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { +pub(crate) fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { #[allow(clippy::useless_conversion)] value.try_into().expect("value should fit in Py_ssize_t") } -#[cfg(test)] -#[cfg(feature = "macros")] -mod tests { +/// Tests specific to the static layout +#[cfg(all(test, feature = "macros"))] +mod static_tests { + use static_assertions::const_assert; + use super::*; use crate::prelude::*; + use memoffset::offset_of; + use std::mem::size_of; + + /// Test the functions calculate properties about the static layout without requiring an instance. + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_type_properties_no_inheritance() { + #[pyclass(crate = "crate", extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + // typically called `ob_base`. In C it is defined using the `PyObject_HEAD` macro + // https://docs.python.org/3/c-api/structures.html + native_base: ffi::PyObject, + contents: PyClassObjectContents, + } - #[pyclass(crate = "crate", subclass)] - struct BaseWithData(#[allow(unused)] u64); + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + let expected_contents_offset = offset_of!(ExpectedLayout, contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_contents_offset), + ); + + let dict_size = size_of::<::Dict>(); + assert_eq!(dict_size, 0); + let expected_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Absolute(expected_contents_offset + expected_dict_offset_in_contents), + ); + + let weakref_size = size_of::<::WeakRef>(); + assert_eq!(weakref_size, 0); + let expected_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Absolute( + expected_contents_offset + expected_weakref_offset_in_contents + ), + ); + + assert_eq!( + expected_dict_offset_in_contents, + expected_weakref_offset_in_contents + ); + } - #[pyclass(crate = "crate", extends = BaseWithData)] - struct ChildWithData(#[allow(unused)] u64); + /// Test the functions calculate properties about the static layout without requiring an instance. + /// The class in this test requires extra space for the `dict` and `weaklist` fields + #[test] + fn test_layout_properties_no_inheritance_optional_fields() { + #[pyclass(crate = "crate", dict, weakref, extends=PyAny)] + struct MyClass(#[allow(unused)] u64); - #[pyclass(crate = "crate", extends = BaseWithData)] - struct ChildWithoutData; + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + // typically called `ob_base`. In C it is defined using the `PyObject_HEAD` macro + // https://docs.python.org/3/c-api/structures.html + native_base: ffi::PyObject, + contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + let expected_contents_offset = offset_of!(ExpectedLayout, contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_contents_offset), + ); + + let dict_size = size_of::<::Dict>(); + assert!(dict_size > 0); + let expected_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Absolute(expected_contents_offset + expected_dict_offset_in_contents), + ); + + let weakref_size = size_of::<::WeakRef>(); + assert!(weakref_size > 0); + let expected_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Absolute( + expected_contents_offset + expected_weakref_offset_in_contents + ), + ); + + assert!(expected_dict_offset_in_contents < expected_weakref_offset_in_contents); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_type_properties_with_inheritance() { + use std::any::TypeId; + + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_field: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_field: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyDictObject, + parent_contents: PyClassObjectContents, + child_contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + Python::with_gil(|py| { + let typ = ChildClass::type_object(py); + let raw_typ = typ.as_ptr().cast::(); + let typ_size = unsafe { (*raw_typ).tp_basicsize }; + assert_eq!(typ_size, expected_size); + }); + + let expected_parent_contents_offset = + offset_of!(ExpectedLayout, parent_contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_parent_contents_offset), + ); + + let expected_child_contents_offset = + offset_of!(ExpectedLayout, child_contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_child_contents_offset), + ); + + let child_dict_size = size_of::<::Dict>(); + assert_eq!(child_dict_size, 0); + let expected_child_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Absolute( + expected_child_contents_offset + expected_child_dict_offset_in_contents + ), + ); + + let child_weakref_size = size_of::<::WeakRef>(); + assert_eq!(child_weakref_size, 0); + let expected_child_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Absolute( + expected_child_contents_offset + expected_child_weakref_offset_in_contents + ), + ); + } + + /// Test the functions that operate on pyclass instances + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_contents_access_no_inheritance() { + #[pyclass(crate = "crate", extends=PyAny)] + struct MyClass { + my_value: u64, + } + + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyObject, + contents: PyClassObjectContents, + } + + Python::with_gil(|py| { + let obj = Py::new(py, MyClass { my_value: 123 }).unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let contents_ptr_int = contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let contents_ptr = contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(contents_ptr, contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let casted_obj = obj_ptr.cast::(); + let expected_contents_ptr = unsafe { addr_of_mut!((*casted_obj).contents) }; + assert_eq!(contents_ptr, expected_contents_ptr); + + // test getting contents by reference + let contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(contents), expected_contents_ptr); + + // test getting data pointer + let data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_data_ptr = unsafe { (*expected_contents_ptr).value.get() }; + assert_eq!(data_ptr, expected_data_ptr); + assert_eq!(unsafe { (*data_ptr).my_value }, 123); + + // test getting data by reference + let data = unsafe { + PyObjectLayout::get_data::(obj.as_raw_ref(), TypeObjectStrategy::lazy(py)) + }; + assert_eq!(ptr_from_ref(data), expected_data_ptr); + }); + } + + /// Test the functions that operate on pyclass instances. + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_contents_access_with_inheritance() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + parent_value: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + child_value: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyDictObject, + parent_contents: PyClassObjectContents, + child_contents: PyClassObjectContents, + } + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass { parent_value: 123 }).add_subclass( + ChildClass { + child_value: "foo".to_owned(), + }, + ), + ) + .unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let parent_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let parent_contents_ptr_int = parent_contents_ptr as usize; + let child_contents_ptr_int = child_contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let parent_contents_ptr = + parent_contents_ptr_int as *mut PyClassObjectContents; + let child_contents_ptr = + child_contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let parent_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + let child_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(parent_contents_ptr, parent_contents_ptr_without_gil); + assert_eq!(child_contents_ptr, child_contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let casted_obj = obj_ptr.cast::(); + let expected_parent_contents_ptr = + unsafe { addr_of_mut!((*casted_obj).parent_contents) }; + let expected_child_contents_ptr = unsafe { addr_of_mut!((*casted_obj).child_contents) }; + assert_eq!(parent_contents_ptr, expected_parent_contents_ptr); + assert_eq!(child_contents_ptr, expected_child_contents_ptr); + + // test getting contents by reference + let parent_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_contents), expected_parent_contents_ptr); + assert_eq!(ptr_from_ref(child_contents), expected_child_contents_ptr); + + // test getting data pointer + let parent_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let child_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_parent_data_ptr = unsafe { (*expected_parent_contents_ptr).value.get() }; + let expected_child_data_ptr = unsafe { (*expected_child_contents_ptr).value.get() }; + assert_eq!(parent_data_ptr, expected_parent_data_ptr); + assert_eq!(unsafe { (*parent_data_ptr).parent_value }, 123); + assert_eq!(child_data_ptr, expected_child_data_ptr); + assert_eq!(unsafe { &(*child_data_ptr).child_value }, "foo"); + + // test getting data by reference + let parent_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_data), expected_parent_data_ptr); + assert_eq!(ptr_from_ref(child_data), expected_child_data_ptr); + }); + } #[test] fn test_inherited_size() { - let base_size = PyObjectLayout::basicsize::(); - assert!(base_size > 0); // negative indicates variable sized - assert_eq!(base_size, PyObjectLayout::basicsize::()); - assert!(base_size < PyObjectLayout::basicsize::()); + #[pyclass(crate = "crate", subclass)] + struct ParentClass; + + #[pyclass(crate = "crate", extends = ParentClass)] + struct ChildClass(#[allow(unused)] u64); + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayoutWithData { + native_base: ffi::PyObject, + parent_class: PyClassObjectContents, + child_class: PyClassObjectContents, + } + let expected_size = std::mem::size_of::() as ffi::Py_ssize_t; + + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + } + + /// Test that `Drop::drop` is called for pyclasses + #[test] + fn test_destructor_called() { + use std::sync::{Arc, Mutex}; + + let deallocations: Arc>> = Arc::new(Mutex::new(Vec::new())); + + #[pyclass(crate = "crate", subclass)] + struct ParentClass(Arc>>); + + impl Drop for ParentClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ParentClass".to_owned()); + } + } + + #[pyclass(crate = "crate", extends = ParentClass)] + struct ChildClass(Arc>>); + + impl Drop for ChildClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ChildClass".to_owned()); + } + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass(deallocations.clone())) + .add_subclass(ChildClass(deallocations.clone())), + ) + .unwrap(); + assert!(deallocations.lock().unwrap().is_empty()); + drop(obj); + }); + + assert_eq!( + deallocations.lock().unwrap().as_slice(), + &["ChildClass", "ParentClass"] + ); } + #[test] + fn test_empty_class() { + #[pyclass(crate = "crate")] + struct EmptyClass; + + // even if the user class has no data some additional space is required + assert!(size_of::>() > 0); + } + + /// It is essential that `InvalidStaticLayout` has 0 size so that it can be distinguished from a valid layout #[test] fn test_invalid_base() { assert_eq!(std::mem::size_of::(), 0); @@ -534,3 +1043,678 @@ mod tests { assert_eq!(std::mem::offset_of!(InvalidLayout, contents), 0); } } + +/// Tests specific to the opaque layout +#[cfg(all(test, Py_3_12, feature = "macros"))] +mod opaque_tests { + use memoffset::offset_of; + use static_assertions::const_assert; + use std::ops::Range; + + use super::*; + + use crate::{prelude::*, PyClass}; + + /// Check that all the type properties are as expected for the given class `T`. + /// Unlike the static layout, the properties of a type in the opaque layout are + /// derived entirely from `T`, not the whole [ffi::PyObject]. + fn check_opaque_type_properties(has_dict: bool, has_weakref: bool) { + let contents_size = size_of::>() as ffi::Py_ssize_t; + // negative indicates 'in addition to the base class' + assert!(PyObjectLayout::basicsize::() == -contents_size); + + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Relative(0), + ); + + let dict_size = size_of::<::Dict>(); + if has_dict { + assert!(dict_size > 0); + } else { + assert_eq!(dict_size, 0); + } + let expected_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Relative(expected_dict_offset_in_contents), + ); + + let weakref_size = size_of::<::WeakRef>(); + if has_weakref { + assert!(weakref_size > 0); + } else { + assert_eq!(weakref_size, 0); + } + let expected_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Relative(expected_weakref_offset_in_contents), + ); + + if has_dict { + assert!(expected_dict_offset_in_contents < expected_weakref_offset_in_contents); + } else { + assert_eq!( + expected_dict_offset_in_contents, + expected_weakref_offset_in_contents + ); + } + } + + /// Test the functions calculate properties about the opaque layout without requiring an instance. + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_type_properties_no_inheritance() { + #[pyclass(crate = "crate", opaque, extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + const_assert!(::OPAQUE == true); + + check_opaque_type_properties::(false, false); + } + + /// Test the functions calculate properties about the opaque layout without requiring an instance. + /// The class in this test requires extra space for the `dict` and `weaklist` fields + #[test] + fn test_layout_properties_no_inheritance_optional_fields() { + #[pyclass(crate = "crate", dict, weakref, opaque, extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + const_assert!(::OPAQUE == true); + + check_opaque_type_properties::(true, true); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_type_properties_with_inheritance_opaque_base() { + use std::any::TypeId; + + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, opaque, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_field: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_field: String, + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + + check_opaque_type_properties::(false, false); + check_opaque_type_properties::(false, false); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_type_properties_with_inheritance_static_base() { + use std::any::TypeId; + + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_field: u64, + } + + #[pyclass(crate = "crate", opaque, extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_field: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == true); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + + check_opaque_type_properties::(false, false); + + #[repr(C)] + struct ParentExpectedLayout { + native_base: ffi::PyDictObject, + parent_contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + let expected_parent_contents_offset = + offset_of!(ParentExpectedLayout, parent_contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_parent_contents_offset), + ); + } + + /// Test the functions that operate on pyclass instances + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_contents_access_no_inheritance() { + #[pyclass(crate = "crate", opaque, extends=PyAny)] + struct MyClass { + my_value: u64, + } + + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new(py, MyClass { my_value: 123 }).unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let contents_ptr_int = contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let contents_ptr = contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(contents_ptr, contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + // the `MyClass` data has to be between the base type and the end of the PyObject. + let pyobject_size = get_pyobject_size::(py); + let contents_range = bytes_range( + contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(contents_range.start >= size_of::()); + assert!(contents_range.end <= pyobject_size); + + // test getting contents by reference + let contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(contents), contents_ptr); + + // test getting data pointer + let data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_data_ptr = unsafe { (*contents_ptr).value.get() }; + assert_eq!(data_ptr, expected_data_ptr); + assert_eq!(unsafe { (*data_ptr).my_value }, 123); + + // test getting data by reference + let data = unsafe { + PyObjectLayout::get_data::(obj.as_raw_ref(), TypeObjectStrategy::lazy(py)) + }; + assert_eq!(ptr_from_ref(data), expected_data_ptr); + }); + } + + /// Test the functions that operate on pyclass instances. + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_contents_access_with_inheritance_opaque_base() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, opaque, extends=PyDict)] + struct ParentClass { + parent_value: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + child_value: String, + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass { parent_value: 123 }).add_subclass( + ChildClass { + child_value: "foo".to_owned(), + }, + ), + ) + .unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let parent_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let parent_contents_ptr_int = parent_contents_ptr as usize; + let child_contents_ptr_int = child_contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let parent_contents_ptr = + parent_contents_ptr_int as *mut PyClassObjectContents; + let child_contents_ptr = + child_contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let parent_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + let child_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(parent_contents_ptr, parent_contents_ptr_without_gil); + assert_eq!(child_contents_ptr, child_contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let parent_pyobject_size = get_pyobject_size::(py); + let child_pyobject_size = get_pyobject_size::(py); + assert!(child_pyobject_size > parent_pyobject_size); + let parent_range = bytes_range( + parent_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + let child_range = bytes_range( + child_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(parent_range.start >= size_of::()); + assert!(parent_range.end <= parent_pyobject_size); + assert!(child_range.start >= parent_range.end); + assert!(child_range.end <= child_pyobject_size); + + // test getting contents by reference + let parent_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_contents), parent_contents_ptr); + assert_eq!(ptr_from_ref(child_contents), child_contents_ptr); + + // test getting data pointer + let parent_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let child_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_parent_data_ptr = unsafe { (*parent_contents_ptr).value.get() }; + let expected_child_data_ptr = unsafe { (*child_contents_ptr).value.get() }; + assert_eq!(parent_data_ptr, expected_parent_data_ptr); + assert_eq!(unsafe { (*parent_data_ptr).parent_value }, 123); + assert_eq!(child_data_ptr, expected_child_data_ptr); + assert_eq!(unsafe { &(*child_data_ptr).child_value }, "foo"); + + // test getting data by reference + let parent_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_data), expected_parent_data_ptr); + assert_eq!(ptr_from_ref(child_data), expected_child_data_ptr); + }); + } + + /// Test the functions that operate on pyclass instances. + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_contents_access_with_inheritance_static_base() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + parent_value: u64, + } + + #[pyclass(crate = "crate", opaque, extends=ParentClass)] + struct ChildClass { + child_value: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass { parent_value: 123 }).add_subclass( + ChildClass { + child_value: "foo".to_owned(), + }, + ), + ) + .unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let parent_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let parent_contents_ptr_int = parent_contents_ptr as usize; + let child_contents_ptr_int = child_contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let parent_contents_ptr = + parent_contents_ptr_int as *mut PyClassObjectContents; + let child_contents_ptr = + child_contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let parent_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + let child_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(parent_contents_ptr, parent_contents_ptr_without_gil); + assert_eq!(child_contents_ptr, child_contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let parent_pyobject_size = get_pyobject_size::(py); + let child_pyobject_size = get_pyobject_size::(py); + assert!(child_pyobject_size > parent_pyobject_size); + let parent_range = bytes_range( + parent_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + let child_range = bytes_range( + child_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(parent_range.start >= size_of::()); + assert!(parent_range.end <= parent_pyobject_size); + assert!(child_range.start >= parent_range.end); + assert!(child_range.end <= child_pyobject_size); + + // test getting contents by reference + let parent_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_contents), parent_contents_ptr); + assert_eq!(ptr_from_ref(child_contents), child_contents_ptr); + + // test getting data pointer + let parent_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let child_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_parent_data_ptr = unsafe { (*parent_contents_ptr).value.get() }; + let expected_child_data_ptr = unsafe { (*child_contents_ptr).value.get() }; + assert_eq!(parent_data_ptr, expected_parent_data_ptr); + assert_eq!(unsafe { (*parent_data_ptr).parent_value }, 123); + assert_eq!(child_data_ptr, expected_child_data_ptr); + assert_eq!(unsafe { &(*child_data_ptr).child_value }, "foo"); + + // test getting data by reference + let parent_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_data), expected_parent_data_ptr); + assert_eq!(ptr_from_ref(child_data), expected_child_data_ptr); + }); + } + + /// Test that `Drop::drop` is called for pyclasses + #[test] + fn test_destructor_called() { + use std::sync::{Arc, Mutex}; + + let deallocations: Arc>> = Arc::new(Mutex::new(Vec::new())); + + #[pyclass(crate = "crate", subclass, opaque)] + struct ParentClass(Arc>>); + + impl Drop for ParentClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ParentClass".to_owned()); + } + } + + #[pyclass(crate = "crate", extends = ParentClass)] + struct ChildClass(Arc>>); + + impl Drop for ChildClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ChildClass".to_owned()); + } + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass(deallocations.clone())) + .add_subclass(ChildClass(deallocations.clone())), + ) + .unwrap(); + assert!(deallocations.lock().unwrap().is_empty()); + drop(obj); + }); + + assert_eq!( + deallocations.lock().unwrap().as_slice(), + &["ChildClass", "ParentClass"] + ); + } + + #[test] + #[should_panic(expected = "OpaqueClassNeverConstructed not initialized")] + fn test_panic_when_incorrectly_assume_initialized() { + #[pyclass(crate = "crate", opaque)] + struct OpaqueClassNeverConstructed; + + const_assert!(::OPAQUE); + + let obj = Python::with_gil(|py| py.None()); + + assert!(OpaqueClassNeverConstructed::try_get_type_object_raw().is_none()); + unsafe { + PyObjectLayout::get_contents_ptr::( + obj.as_ptr(), + TypeObjectStrategy::assume_init(), + ); + } + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic(expected = "the object is not an instance of")] + fn test_panic_when_incorrect_type() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, opaque, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_value: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_value: String, + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new(py, ParentClass { parent_value: 123 }).unwrap(); + let obj_ptr = obj.as_ptr(); + + unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + }); + } + + /// The size in bytes of a [ffi::PyObject] of the type `T` + fn get_pyobject_size(py: Python<'_>) -> usize { + let typ = ::type_object(py); + let raw_typ = typ.as_ptr().cast::(); + let size = unsafe { (*raw_typ).tp_basicsize }; + usize::try_from(size).expect("size should be a valid usize") + } + + /// Create a range from a start and size instead of a start and end + fn bytes_range(start: usize, size: usize) -> Range { + Range { + start, + end: start + size, + } + } +} + +#[cfg(all(test, not(Py_3_12), feature = "macros"))] +mod opaque_fail_tests { + use crate::{ + prelude::*, + types::{PyDict, PyTuple, PyType}, + PyTypeInfo, + }; + + #[pyclass(crate = "crate", extends=PyType)] + #[derive(Default)] + struct Metaclass; + + #[pymethods(crate = "crate")] + impl Metaclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__(&mut self, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) {} + } + + /// PyType uses the opaque layout. Explicitly using `#[pyclass(opaque)]` can be caught at compile time + /// but it is possible to create a pyclass that uses the opaque layout by extending an opaque native type. + #[test] + #[should_panic( + expected = "The opaque object layout (used by pyo3::pycell::layout::opaque_fail_tests::Metaclass) is not supported until python 3.12" + )] + fn test_panic_at_construction_inherit_opaque() { + Python::with_gil(|py| { + Py::new(py, Metaclass::default()).unwrap(); + }); + } + + #[test] + #[should_panic( + expected = "The opaque object layout (used by pyo3::pycell::layout::opaque_fail_tests::Metaclass) is not supported until python 3.12" + )] + fn test_panic_at_type_construction_inherit_opaque() { + Python::with_gil(|py| { + ::type_object(py); + }); + } +} diff --git a/src/type_object.rs b/src/type_object.rs index 2f952699382..13b7c3c732e 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -56,15 +56,15 @@ pub unsafe trait PyTypeInfo: Sized { /// of a new `repr(C)` struct const OPAQUE: bool; - /// Returns the `PyTypeObject` instance for this type. + /// Returns the [ffi::PyTypeObject] instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; - /// Returns the `PyTypeObject` instance for this type if it is known statically or has already - /// been initialized (by calling `type_object_raw()`). + /// Returns the [ffi::PyTypeObject] instance for this type if it is known statically or has already + /// been initialized (by calling [PyTypeInfo::type_object_raw()]). /// /// # Safety /// - It is valid to always return Some. - /// - It is not valid to return None once `type_object_raw()` has been called. + /// - It is not valid to return None once [PyTypeInfo::type_object_raw()] has been called. fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject>; /// Returns the safe abstraction over the type object. diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 6bdac4a5f95..92afc2cfd1c 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -645,20 +645,18 @@ fn access_frozen_class_without_gil() { } let py_counter: Py = Python::with_gil(|py| { - Py::::enable_get(py); - let counter = FrozenCounter { value: AtomicUsize::new(0), }; let cell = Bound::new(py, counter).unwrap(); - unsafe { cell.get().value.fetch_add(1, Ordering::Relaxed) }; + cell.get().value.fetch_add(1, Ordering::Relaxed); cell.into() }); - assert_eq!(unsafe { py_counter.get().value.load(Ordering::Relaxed) }, 1); + assert_eq!(py_counter.get().value.load(Ordering::Relaxed), 1); Python::with_gil(move |_py| drop(py_counter)); } diff --git a/tests/test_class_init.rs b/tests/test_class_init.rs index 175c01e5971..7025a9aff00 100644 --- a/tests/test_class_init.rs +++ b/tests/test_class_init.rs @@ -292,6 +292,7 @@ impl SubClass { } #[test] +#[should_panic(expected = "initialize_with_default does not currently support multi-level inheritance")] fn subclass_pyclass_init() { Python::with_gil(|py| { let sub_cls = py.get_type::(); diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index ff538788427..fd1938de742 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -23,9 +23,8 @@ fn test_compile_errors() { t.compile_fail("tests/ui/reject_generics.rs"); t.compile_fail("tests/ui/deprecations.rs"); t.compile_fail("tests/ui/invalid_closure.rs"); - // only possible to extend variable sized types after 3.12 #[cfg(not(Py_3_12))] - t.compile_fail("tests/ui/invalid_extend_variable_sized.rs"); + t.compile_fail("tests/ui/invalid_opaque.rs"); t.compile_fail("tests/ui/pyclass_send.rs"); t.compile_fail("tests/ui/invalid_argument_attributes.rs"); t.compile_fail("tests/ui/invalid_intopy_derive.rs"); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index d308cf1be06..e48401bceb2 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -220,12 +220,12 @@ mod inheriting_type { } #[test] - fn inherit_type() { + fn test_metaclass() { Python::with_gil(|py| { #[allow(non_snake_case)] let Metaclass = py.get_type::(); - // checking base is `type` + // check base type py_run!(py, Metaclass, r#"assert Metaclass.__bases__ == (type,)"#); // check can be used as a metaclass @@ -281,6 +281,50 @@ mod inheriting_type { }); } + #[pyclass(subclass, extends=Metaclass)] + #[derive(Debug)] + struct MetaclassSubclass { + subclass_value: String, + } + + impl Default for MetaclassSubclass { + fn default() -> Self { + Self { + subclass_value: "foo".to_owned(), + } + } + } + + #[pymethods] + impl MetaclassSubclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + _slf: Bound<'_, MetaclassSubclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + } + } + + #[test] + #[should_panic( + expected = "initialize_with_default does not currently support multi-level inheritance" + )] + fn test_metaclass_subclass() { + Python::with_gil(|py| { + #[allow(non_snake_case)] + let MetaclassSubclass = py.get_type::(); + py_run!( + py, + MetaclassSubclass, + r#" + class Foo(metaclass=MetaclassSubclass): + pass + "# + ); + }); + } + #[test] #[should_panic(expected = "Metaclasses must specify __init__")] fn inherit_type_missing_init() { @@ -288,7 +332,7 @@ mod inheriting_type { #[pyclass(subclass, extends=PyType)] #[derive(Debug, Default)] - struct MetaclassMissingInit {} + struct MetaclassMissingInit; #[pymethods] impl MetaclassMissingInit {} @@ -316,7 +360,7 @@ mod inheriting_type { #[pyclass(subclass, extends=PyType)] #[derive(Debug, Default)] - struct MetaclassWithNew {} + struct MetaclassWithNew; #[pymethods] impl MetaclassWithNew { diff --git a/tests/ui/invalid_extend_variable_sized.rs b/tests/ui/invalid_extend_variable_sized.rs deleted file mode 100644 index 1ebd04a1721..00000000000 --- a/tests/ui/invalid_extend_variable_sized.rs +++ /dev/null @@ -1,14 +0,0 @@ -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyTuple, PyType}; - -#[pyclass(extends=PyType)] -#[derive(Default)] -struct MyClass {} - -#[pymethods] -impl MyClass { - #[pyo3(signature = (*_args, **_kwargs))] - fn __init__(&mut self, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) {} -} - -fn main() {} diff --git a/tests/ui/invalid_extend_variable_sized.stderr b/tests/ui/invalid_extend_variable_sized.stderr deleted file mode 100644 index 5f25623d5e7..00000000000 --- a/tests/ui/invalid_extend_variable_sized.stderr +++ /dev/null @@ -1,15 +0,0 @@ -error[E0277]: the class layout is not valid - --> tests/ui/invalid_extend_variable_sized.rs:4:1 - | -4 | #[pyclass(extends=PyType)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=...)]` - | - = help: the trait `pyo3::pycell::impl_::PyClassObjectLayout` is not implemented for `PyVariableClassObject` - = note: the python version being built against influences which layouts are valid - = help: the trait `pyo3::pycell::impl_::PyClassObjectLayout` is implemented for `PyStaticClassObject` -note: required by a bound in `PyClassImpl::Layout` - --> src/impl_/pyclass.rs - | - | type Layout: PyClassObjectLayout; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::Layout` - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/invalid_frozen_pyclass_borrow.rs b/tests/ui/invalid_frozen_pyclass_borrow.rs index 4846080676d..6379a8707c5 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.rs +++ b/tests/ui/invalid_frozen_pyclass_borrow.rs @@ -26,11 +26,11 @@ fn borrow_mut_of_child_fails(child: Py, py: Python) { } fn py_get_of_mutable_class_fails(class: Py) { - unsafe { class.get(); } + class.get(); } fn pyclass_get_of_mutable_class_fails(class: &Bound<'_, MutableBase>) { - unsafe { class.get(); } + class.get(); } #[pyclass(frozen)] diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index 2d2b7deb6ce..52a0623f282 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -60,31 +60,31 @@ note: required by a bound in `pyo3::Bound::<'py, T>::borrow_mut` | ^^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:20 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:11 | -29 | unsafe { class.get(); } - | ^^^ expected `True`, found `False` +29 | class.get(); + | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::Py::::get` --> src/instance.rs | - | pub unsafe fn get(&self) -> &T - | --- required by a bound in this associated function + | pub fn get(&self) -> &T + | --- required by a bound in this associated function | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `Py::::get` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:33:20 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:33:11 | -33 | unsafe { class.get(); } - | ^^^ expected `True`, found `False` +33 | class.get(); + | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::Bound::<'py, T>::get` --> src/instance.rs | - | pub unsafe fn get(&self) -> &T - | --- required by a bound in this associated function + | pub fn get(&self) -> &T + | --- required by a bound in this associated function | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::get` diff --git a/tests/ui/invalid_opaque.rs b/tests/ui/invalid_opaque.rs new file mode 100644 index 00000000000..8f4d72bb0d6 --- /dev/null +++ b/tests/ui/invalid_opaque.rs @@ -0,0 +1,6 @@ +use pyo3::prelude::*; + +#[pyclass(opaque)] +struct MyClass; + +fn main() {} diff --git a/tests/ui/invalid_opaque.stderr b/tests/ui/invalid_opaque.stderr new file mode 100644 index 00000000000..96b16507365 --- /dev/null +++ b/tests/ui/invalid_opaque.stderr @@ -0,0 +1,16 @@ +error: #[pyclass(opaque)] requires python 3.12 or later + --> tests/ui/invalid_opaque.rs:3:1 + | +3 | #[pyclass(opaque)] + | ^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0046]: not all trait items implemented, missing: `OPAQUE` + --> tests/ui/invalid_opaque.rs:3:1 + | +3 | #[pyclass(opaque)] + | ^^^^^^^^^^^^^^^^^^ missing `OPAQUE` in implementation + | + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) + = help: implement the missing item: `const OPAQUE: bool = false;` diff --git a/tests/ui/invalid_pyclass_args.stderr b/tests/ui/invalid_pyclass_args.stderr index 15aa0387cc6..72894a69aa1 100644 --- a/tests/ui/invalid_pyclass_args.stderr +++ b/tests/ui/invalid_pyclass_args.stderr @@ -1,4 +1,4 @@ -error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` +error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `opaque`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` --> tests/ui/invalid_pyclass_args.rs:4:11 | 4 | #[pyclass(extend=pyo3::types::PyDict)] @@ -46,7 +46,7 @@ error: expected string literal 25 | #[pyclass(module = my_module)] | ^^^^^^^^^ -error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` +error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `opaque`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` --> tests/ui/invalid_pyclass_args.rs:28:11 | 28 | #[pyclass(weakrev)] From d24dbc46836d3fcd280891db2656e5f65c0afc7f Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 22 Nov 2024 23:00:36 +0000 Subject: [PATCH 38/41] fix tests --- guide/src/class.md | 10 +++- src/impl_/coroutine.rs | 59 +++++++++++++-------- src/impl_/pyclass_init.rs | 11 ++-- src/pycell/borrow_checker.rs | 2 +- src/pycell/layout.rs | 53 +++++++++++------- tests/test_class_init.rs | 4 +- tests/test_inheritance.rs | 14 +---- tests/ui/abi3_inheritance.stderr | 20 +++---- tests/ui/abi3_nativetype_inheritance.stderr | 20 +++---- 9 files changed, 110 insertions(+), 83 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 091b4fb3809..b4a9854d880 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1378,13 +1378,20 @@ impl pyo3::types::DerefToPyAny for MyClass {} unsafe impl pyo3::type_object::PyTypeInfo for MyClass { const NAME: &'static str = "MyClass"; const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None; - type Layout = pyo3::impl_::pycell::PyStaticClassObject; + const OPAQUE: bool = false; + #[inline] fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject { ::lazy_type_object() .get_or_init(py) .as_type_ptr() } + + #[inline] + fn try_get_type_object_raw() -> ::std::option::Option<*mut pyo3::ffi::PyTypeObject> { + ::lazy_type_object() + .try_get_raw() + } } impl pyo3::PyClass for MyClass { @@ -1423,7 +1430,6 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { const IS_SUBCLASS: bool = false; const IS_MAPPING: bool = false; const IS_SEQUENCE: bool = false; - type Layout = ::Layout; type BaseType = PyAny; type ThreadChecker = pyo3::impl_::pyclass::SendablePyClass; type PyClassMutability = <::PyClassMutability as pyo3::impl_::pycell::PyClassMutability>::MutableChild; diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index 2c7924fffd2..6d53d11ff1c 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -6,13 +6,17 @@ use std::{ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, - pycell::borrow_checker::PyClassBorrowChecker, - pycell::layout::PyClassObjectLayout, + pycell::{ + borrow_checker::PyClassBorrowChecker, + layout::{PyObjectLayout, TypeObjectStrategy}, + }, pyclass::boolean_struct::False, types::{PyAnyMethods, PyString}, IntoPyObject, Py, PyAny, PyClass, PyErr, PyResult, Python, }; +use super::pycell::GetBorrowChecker; + pub fn new_coroutine<'py, F, T, E>( name: &Bound<'py, PyString>, qualname_prefix: Option<&'static str>, @@ -27,16 +31,15 @@ where Coroutine::new(Some(name.clone()), qualname_prefix, throw_callback, future) } -fn get_ptr(obj: &Py) -> *mut T { - obj.get_class_object().get_ptr() -} - pub struct RefGuard(Py); impl RefGuard { pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { let bound = obj.downcast::()?; - bound.get_class_object().borrow_checker().try_borrow()?; + // SAFETY: can assume the type object for `T` is initialized because an instance (`obj`) has been created. + let strategy = unsafe { TypeObjectStrategy::assume_init() }; + let borrow_checker = T::PyClassMutability::borrow_checker(obj.as_raw_ref(), strategy); + borrow_checker.try_borrow()?; Ok(RefGuard(bound.clone().unbind())) } } @@ -45,18 +48,19 @@ impl Deref for RefGuard { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: `RefGuard` has been built from `PyRef` and provides the same guarantees - unsafe { &*get_ptr(&self.0) } + unsafe { + PyObjectLayout::get_data::(self.0.as_raw_ref(), TypeObjectStrategy::assume_init()) + } } } impl Drop for RefGuard { fn drop(&mut self) { - Python::with_gil(|gil| { - self.0 - .bind(gil) - .get_class_object() - .borrow_checker() - .release_borrow() + Python::with_gil(|py| { + // SAFETY: `self.0` contains an object that is an instance of `T` + let borrow_checker = + unsafe { PyObjectLayout::get_borrow_checker::(py, self.0.as_raw_ref()) }; + borrow_checker.release_borrow(); }) } } @@ -66,7 +70,10 @@ pub struct RefMutGuard>(Py); impl> RefMutGuard { pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { let bound = obj.downcast::()?; - bound.get_class_object().borrow_checker().try_borrow_mut()?; + // SAFETY: can assume the type object for `T` is initialized because an instance (`obj`) has been created. + let strategy = unsafe { TypeObjectStrategy::assume_init() }; + let borrow_checker = T::PyClassMutability::borrow_checker(obj.as_raw_ref(), strategy); + borrow_checker.try_borrow_mut()?; Ok(RefMutGuard(bound.clone().unbind())) } } @@ -75,25 +82,31 @@ impl> Deref for RefMutGuard { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: `RefMutGuard` has been built from `PyRefMut` and provides the same guarantees - unsafe { &*get_ptr(&self.0) } + unsafe { + PyObjectLayout::get_data::(self.0.as_raw_ref(), TypeObjectStrategy::assume_init()) + } } } impl> DerefMut for RefMutGuard { fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: `RefMutGuard` has been built from `PyRefMut` and provides the same guarantees - unsafe { &mut *get_ptr(&self.0) } + unsafe { + &mut *PyObjectLayout::get_data_ptr::( + self.0.as_ptr(), + TypeObjectStrategy::assume_init(), + ) + } } } impl> Drop for RefMutGuard { fn drop(&mut self) { - Python::with_gil(|gil| { - self.0 - .bind(gil) - .get_class_object() - .borrow_checker() - .release_borrow_mut() + Python::with_gil(|py| { + // SAFETY: `self.0` contains an object that is an instance of `T` + let borrow_checker = + unsafe { PyObjectLayout::get_borrow_checker::(py, self.0.as_raw_ref()) }; + borrow_checker.release_borrow_mut(); }) } } diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index caf9e8d9619..b31657867f3 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -12,11 +12,12 @@ pub unsafe fn initialize_with_default( py: Python<'_>, obj: *mut ffi::PyObject, ) { - if TypeId::of::() != TypeId::of::() { - // only sets the PyClassContents of the 'most derived type' - // so any parent pyclasses would remain uninitialized. - panic!("initialize_with_default does not currently support multi-level inheritance"); - } + // only sets the PyClassContents of the 'most derived type' + // so any parent pyclasses would remain uninitialized. + assert!( + TypeId::of::() == TypeId::of::(), + "initialize_with_default does not currently support multi-level inheritance" + ); std::ptr::write( PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)), PyClassObjectContents::new(T::default()), diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index 2673c9d3a32..5569129a652 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -163,7 +163,7 @@ impl PyClassBorrowChecker for BorrowChecker { } fn release_borrow_mut(&self) { - self.0.0.store(BorrowFlag::UNUSED, Ordering::Release) + self.0 .0.store(BorrowFlag::UNUSED, Ordering::Release) } } diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 2acf899feff..6cd51963fc3 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -546,9 +546,12 @@ pub(crate) fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { /// Tests specific to the static layout #[cfg(all(test, feature = "macros"))] +#[allow(clippy::bool_comparison)] // `== false` is harder to miss than ! mod static_tests { use static_assertions::const_assert; + #[cfg(not(Py_LIMITED_API))] + use super::test_utils::get_pyobject_size; use super::*; use crate::prelude::*; @@ -697,9 +700,7 @@ mod static_tests { assert_eq!(PyObjectLayout::basicsize::(), expected_size); Python::with_gil(|py| { - let typ = ChildClass::type_object(py); - let raw_typ = typ.as_ptr().cast::(); - let typ_size = unsafe { (*raw_typ).tp_basicsize }; + let typ_size = get_pyobject_size::(py) as isize; assert_eq!(typ_size, expected_size); }); @@ -1046,11 +1047,14 @@ mod static_tests { /// Tests specific to the opaque layout #[cfg(all(test, Py_3_12, feature = "macros"))] +#[allow(clippy::bool_comparison)] // `== false` is harder to miss than ! mod opaque_tests { use memoffset::offset_of; use static_assertions::const_assert; use std::ops::Range; + #[cfg(not(Py_LIMITED_API))] + use super::test_utils::get_pyobject_size; use super::*; use crate::{prelude::*, PyClass}; @@ -1250,13 +1254,16 @@ mod opaque_tests { // test that contents pointer matches expecations // the `MyClass` data has to be between the base type and the end of the PyObject. - let pyobject_size = get_pyobject_size::(py); - let contents_range = bytes_range( - contents_ptr_int - obj_ptr_int, - size_of::>(), - ); - assert!(contents_range.start >= size_of::()); - assert!(contents_range.end <= pyobject_size); + #[cfg(not(Py_LIMITED_API))] + { + let pyobject_size = get_pyobject_size::(py); + let contents_range = bytes_range( + contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(contents_range.start >= size_of::()); + assert!(contents_range.end <= pyobject_size); + } // test getting contents by reference let contents = unsafe { @@ -1628,7 +1635,7 @@ mod opaque_tests { } #[test] - #[cfg(debug_assertions)] + #[cfg(all(debug_assertions, not(Py_LIMITED_API)))] #[should_panic(expected = "the object is not an instance of")] fn test_panic_when_incorrect_type() { use crate::types::PyDict; @@ -1661,15 +1668,8 @@ mod opaque_tests { }); } - /// The size in bytes of a [ffi::PyObject] of the type `T` - fn get_pyobject_size(py: Python<'_>) -> usize { - let typ = ::type_object(py); - let raw_typ = typ.as_ptr().cast::(); - let size = unsafe { (*raw_typ).tp_basicsize }; - usize::try_from(size).expect("size should be a valid usize") - } - /// Create a range from a start and size instead of a start and end + #[allow(unused)] fn bytes_range(start: usize, size: usize) -> Range { Range { start, @@ -1718,3 +1718,18 @@ mod opaque_fail_tests { }); } } + +#[cfg(test)] +mod test_utils { + #[cfg(not(Py_LIMITED_API))] + use crate::{ffi, PyClass, PyTypeInfo, Python}; + + /// The size in bytes of a [ffi::PyObject] of the type `T` + #[cfg(not(Py_LIMITED_API))] + pub fn get_pyobject_size(py: Python<'_>) -> usize { + let typ = ::type_object(py); + let raw_typ = typ.as_ptr().cast::(); + let size = unsafe { (*raw_typ).tp_basicsize }; + usize::try_from(size).expect("size should be a valid usize") + } +} diff --git a/tests/test_class_init.rs b/tests/test_class_init.rs index 7025a9aff00..658fd120f39 100644 --- a/tests/test_class_init.rs +++ b/tests/test_class_init.rs @@ -292,7 +292,9 @@ impl SubClass { } #[test] -#[should_panic(expected = "initialize_with_default does not currently support multi-level inheritance")] +#[should_panic( + expected = "initialize_with_default does not currently support multi-level inheritance" +)] fn subclass_pyclass_init() { Python::with_gil(|py| { let sub_cls = py.get_type::(); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index e48401bceb2..7fca4138c32 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -282,18 +282,8 @@ mod inheriting_type { } #[pyclass(subclass, extends=Metaclass)] - #[derive(Debug)] - struct MetaclassSubclass { - subclass_value: String, - } - - impl Default for MetaclassSubclass { - fn default() -> Self { - Self { - subclass_value: "foo".to_owned(), - } - } - } + #[derive(Debug, Default)] + struct MetaclassSubclass {} #[pymethods] impl MetaclassSubclass { diff --git a/tests/ui/abi3_inheritance.stderr b/tests/ui/abi3_inheritance.stderr index 1059c1e4b5c..110092cb0a8 100644 --- a/tests/ui/abi3_inheritance.stderr +++ b/tests/ui/abi3_inheritance.stderr @@ -1,8 +1,8 @@ error[E0277]: pyclass `PyException` cannot be subclassed - --> tests/ui/abi3_inheritance.rs:4:1 + --> tests/ui/abi3_inheritance.rs:4:19 | 4 | #[pyclass(extends=PyException)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` + | ^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` | = help: the trait `PyClassBaseType` is not implemented for `PyException` = note: `PyException` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -10,13 +10,17 @@ error[E0277]: pyclass `PyException` cannot be subclassed = help: the following other types implement trait `PyClassBaseType`: PyAny PyType - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` error[E0277]: pyclass `PyException` cannot be subclassed - --> tests/ui/abi3_inheritance.rs:4:19 + --> tests/ui/abi3_inheritance.rs:4:1 | 4 | #[pyclass(extends=PyException)] - | ^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyException)]` | = help: the trait `PyClassBaseType` is not implemented for `PyException` = note: `PyException` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -24,8 +28,4 @@ error[E0277]: pyclass `PyException` cannot be subclassed = help: the following other types implement trait `PyClassBaseType`: PyAny PyType -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index a6886a6b906..6a8d9f29c1c 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -1,8 +1,8 @@ error[E0277]: pyclass `PyDict` cannot be subclassed - --> tests/ui/abi3_nativetype_inheritance.rs:5:1 + --> tests/ui/abi3_nativetype_inheritance.rs:5:19 | 5 | #[pyclass(extends=PyDict)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyDict)]` + | ^^^^^^ required for `#[pyclass(extends=PyDict)]` | = help: the trait `PyClassBaseType` is not implemented for `PyDict` = note: `PyDict` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -10,13 +10,17 @@ error[E0277]: pyclass `PyDict` cannot be subclassed = help: the following other types implement trait `PyClassBaseType`: PyAny PyType - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` error[E0277]: pyclass `PyDict` cannot be subclassed - --> tests/ui/abi3_nativetype_inheritance.rs:5:19 + --> tests/ui/abi3_nativetype_inheritance.rs:5:1 | 5 | #[pyclass(extends=PyDict)] - | ^^^^^^ required for `#[pyclass(extends=PyDict)]` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required for `#[pyclass(extends=PyDict)]` | = help: the trait `PyClassBaseType` is not implemented for `PyDict` = note: `PyDict` must have `#[pyclass(subclass)]` to be eligible for subclassing @@ -24,8 +28,4 @@ error[E0277]: pyclass `PyDict` cannot be subclassed = help: the following other types implement trait `PyClassBaseType`: PyAny PyType -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) From 53a13e1869133dd32b9159befba7bbce0f9e1d1e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 22 Nov 2024 23:20:32 +0000 Subject: [PATCH 39/41] fixes --- Cargo.toml | 4 +-- newsfragments/4678.added.md | 2 +- pyo3-macros-backend/src/pyclass.rs | 2 +- src/exceptions.rs | 3 +- src/internal_tricks.rs | 12 +++++++ src/pycell/layout.rs | 41 ++++++++++++----------- src/types/set.rs | 1 - tests/test_variable_sized_class_basics.rs | 4 +-- 8 files changed, 41 insertions(+), 28 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d7eb03d55d4..42dad19ed8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.23.1" +version = "0.24.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -12,7 +12,7 @@ categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/noxfile.py", "/.github", "/tests/test_compile_error.rs", "/tests/ui"] edition = "2021" -rust-version = "1.65" +rust-version = "1.63" [dependencies] cfg-if = "1.0" diff --git a/newsfragments/4678.added.md b/newsfragments/4678.added.md index 8520219a34f..1d8f3293133 100644 --- a/newsfragments/4678.added.md +++ b/newsfragments/4678.added.md @@ -1 +1 @@ -Add support for extending variable/unknown sized base classes (eg `type` to create metaclasses) \ No newline at end of file +Add support for opaque PyObjects allowing extending variable/unknown sized base classes (including `type` to create metaclasses) \ No newline at end of file diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 06a39bab4c5..042b6ebb368 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1198,7 +1198,7 @@ fn impl_complex_enum_variant_match_args( }); parse_quote! { #[allow(non_upper_case_globals)] - const __match_args__: ( #(#args_tp,)* ) = ( + const #ident: ( #(#args_tp,)* ) = ( #(stringify!(#field_names),)* ); } diff --git a/src/exceptions.rs b/src/exceptions.rs index 4f358ced1f7..d3766ad61f4 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -276,7 +276,8 @@ macro_rules! impl_windows_native_exception ( pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, |_py| unsafe { $crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject }); + $crate::pyobject_native_type!($name, $layout); + $crate::pyobject_native_type_object_methods!($name, #global_ptr=$crate::ffi::$exc_name); ); ($name:ident, $exc_name:ident, $doc:expr) => ( impl_windows_native_exception!($name, $exc_name, $doc, $crate::ffi::PyBaseExceptionObject); diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 97b13aff2a8..d2e67326c11 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -47,3 +47,15 @@ pub(crate) const fn ptr_from_ref(t: &T) -> *const T { pub(crate) fn ptr_from_mut(t: &mut T) -> *mut T { t as *mut T } + +// TODO: use ptr::cast_mut on MSRV 1.65 +#[inline] +pub(crate) fn cast_mut(t: *const T) -> *mut T { + t as *mut T +} + +// TODO: use ptr::cast_const on MSRV 1.65 +#[inline] +pub(crate) fn cast_const(t: *mut T) -> *const T { + t as *const T +} diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index 6cd51963fc3..a734154231d 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -6,10 +6,13 @@ use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ptr::addr_of_mut; +use memoffset::offset_of; + use crate::impl_::pyclass::{ PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, }; use crate::internal::get_slot::{TP_DEALLOC, TP_FREE}; +use crate::internal_tricks::{cast_const, cast_mut}; use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; use crate::type_object::PyNativeType; use crate::types::PyType; @@ -175,6 +178,7 @@ impl PyObjectRecursiveOperations .get_slot(TP_DEALLOC) .expect("PyType_Type should have tp_dealloc"); // `PyType_Type::dealloc` calls `Py_GC_UNTRACK` so we have to re-track before deallocating + #[cfg(not(PyPy))] ffi::PyObject_GC_Track(obj.cast()); return tp_dealloc(obj.cast()); } @@ -214,7 +218,7 @@ pub(crate) mod opaque_layout { use crate::{impl_::pyclass::PyClassImpl, PyTypeInfo}; #[cfg(Py_3_12)] - pub fn get_contents_ptr( + pub(crate) fn get_contents_ptr( obj: *mut ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { @@ -356,7 +360,7 @@ impl PyObjectLayout { let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); // indicates `ob_base` has type InvalidBaseLayout debug_assert_ne!( - std::mem::offset_of!(static_layout::PyStaticClassLayout, contents), + offset_of!(static_layout::PyStaticClassLayout, contents), 0, "invalid ob_base found" ); @@ -372,10 +376,10 @@ impl PyObjectLayout { obj: &'a ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> &'a PyClassObjectContents { - unsafe { - &*PyObjectLayout::get_contents_ptr::(ptr_from_ref(obj).cast_mut(), strategy) - .cast_const() - } + &*cast_const(PyObjectLayout::get_contents_ptr::( + cast_mut(ptr_from_ref(obj)), + strategy, + )) } /// Obtain a pointer to the portion of `obj` containing the data for `T` @@ -398,7 +402,7 @@ impl PyObjectLayout { obj: &'a ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> &'a T { - unsafe { &*PyObjectLayout::get_data_ptr::(ptr_from_ref(obj).cast_mut(), strategy) } + &*PyObjectLayout::get_data_ptr::(cast_mut(ptr_from_ref(obj)), strategy) } /// Obtain a reference to the borrow checker for `obj` @@ -436,9 +440,7 @@ impl PyObjectLayout { py: Python<'_>, obj: *mut ffi::PyObject, ) { - unsafe { - PyClassRecursiveOperations::::deallocate(py, obj); - }; + PyClassRecursiveOperations::::deallocate(py, obj); } /// Clean up then free the memory associated with `obj`. @@ -450,13 +452,11 @@ impl PyObjectLayout { py: Python<'_>, obj: *mut ffi::PyObject, ) { - unsafe { - #[cfg(not(PyPy))] - { - ffi::PyObject_GC_UnTrack(obj.cast()); - } - PyClassRecursiveOperations::::deallocate(py, obj); - }; + #[cfg(not(PyPy))] + { + ffi::PyObject_GC_UnTrack(obj.cast()); + } + PyClassRecursiveOperations::::deallocate(py, obj); } /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` @@ -613,6 +613,7 @@ mod static_tests { /// Test the functions calculate properties about the static layout without requiring an instance. /// The class in this test requires extra space for the `dict` and `weaklist` fields #[test] + #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] fn test_layout_properties_no_inheritance_optional_fields() { #[pyclass(crate = "crate", dict, weakref, extends=PyAny)] struct MyClass(#[allow(unused)] u64); @@ -1041,7 +1042,7 @@ mod static_tests { contents: u8, } - assert_eq!(std::mem::offset_of!(InvalidLayout, contents), 0); + assert_eq!(offset_of!(InvalidLayout, contents), 0); } } @@ -1051,6 +1052,7 @@ mod static_tests { mod opaque_tests { use memoffset::offset_of; use static_assertions::const_assert; + use std::mem::size_of; use std::ops::Range; #[cfg(not(Py_LIMITED_API))] @@ -1704,7 +1706,7 @@ mod opaque_fail_tests { )] fn test_panic_at_construction_inherit_opaque() { Python::with_gil(|py| { - Py::new(py, Metaclass::default()).unwrap(); + Py::new(py, Metaclass).unwrap(); }); } @@ -1726,6 +1728,7 @@ mod test_utils { /// The size in bytes of a [ffi::PyObject] of the type `T` #[cfg(not(Py_LIMITED_API))] + #[allow(unused)] pub fn get_pyobject_size(py: Python<'_>) -> usize { let typ = ::type_object(py); let raw_typ = typ.as_ptr().cast::(); diff --git a/src/types/set.rs b/src/types/set.rs index 87f6749ab97..51ced5fac5b 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -29,7 +29,6 @@ pyobject_native_type!(PySet, ffi::PySetObject, #checkfunction=ffi::PySet_Check); #[cfg(any(PyPy, GraalPy))] pyobject_native_type_core!(PySet, #checkfunction=ffi::PySet_Check); -#[cfg(not(any(PyPy, GraalPy)))] pyobject_native_type_object_methods!(PySet, #global=ffi::PySet_Type); impl PySet { diff --git a/tests/test_variable_sized_class_basics.rs b/tests/test_variable_sized_class_basics.rs index e190e400ef6..9adde34d849 100644 --- a/tests/test_variable_sized_class_basics.rs +++ b/tests/test_variable_sized_class_basics.rs @@ -45,9 +45,7 @@ fn class_with_object_field() { let obj = Bound::new(py, ClassWithObjectField { value: None }).unwrap(); py_run!(py, obj, "obj.value = 5"); let obj_ref = obj.borrow(); - let Some(value) = &obj_ref.value else { - panic!("obj_ref.value is None"); - }; + let value = obj_ref.value.as_ref().unwrap(); assert_eq!(*value.downcast_bound::(py).unwrap(), 5); }); } From c97b64b36b93209c8f6ac287e81ddebb0723ac27 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 23 Nov 2024 01:22:11 +0000 Subject: [PATCH 40/41] documentation changes --- src/impl_/pyclass.rs | 4 ++-- src/impl_/trampoline.rs | 4 ++-- src/instance.rs | 1 - src/pycell/layout.rs | 24 ++++++++++++------------ src/type_object.rs | 2 +- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index e9a958792bc..965871d33de 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1146,8 +1146,8 @@ impl PyClassThreadChecker for ThreadCheckerImpl { ) )] pub trait PyClassBaseType: Sized { - /// A struct that describes the memory layout of a `*mut ffi:PyObject` with the type of `Self`. - /// Only valid when `::OPAQUE` is false. + /// A struct that describes the memory layout of a `ffi:PyObject` with the type of `Self`. + /// Only valid when `::OPAQUE` is `false`. type StaticLayout: PyLayout; type BaseNativeType: PyTypeInfo; type RecursiveOperations: PyObjectRecursiveOperations; diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 3af8eaa4a07..32416af6a02 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -122,8 +122,8 @@ trampolines!( pub fn unaryfunc(slf: *mut ffi::PyObject) -> *mut ffi::PyObject; ); -// tp_init should return 0 on success and -1 on error -// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_init +/// `tp_init` should return 0 on success and -1 on error. +/// [docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_init) #[inline] pub unsafe fn initproc( slf: *mut ffi::PyObject, diff --git a/src/instance.rs b/src/instance.rs index eb0168fb4fe..a7b138a2a8b 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1286,7 +1286,6 @@ where /// /// This is available if the class is [`frozen`][macro@crate::pyclass] and [`Sync`]. /// - /// /// # Examples /// /// ``` diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index a734154231d..f651f0f6b13 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -36,7 +36,7 @@ pub(crate) struct PyClassObjectContents { pub(crate) value: ManuallyDrop>, pub(crate) borrow_checker: ::Storage, pub(crate) thread_checker: T::ThreadChecker, - /// A pointer to a [ffi::PyObject]` if `T` is annotated with `#[pyclass(dict)]` and a zero-sized field otherwise. + /// A pointer to a [ffi::PyObject] if `T` is annotated with `#[pyclass(dict)]` and a zero-sized field otherwise. pub(crate) dict: T::Dict, /// A pointer to a [ffi::PyObject] if `T` is annotated with `#[pyclass(weakref)]` and a zero-sized field otherwise. pub(crate) weakref: T::WeakRef, @@ -62,7 +62,7 @@ impl PyClassObjectContents { } } -/// Functions for working with `PyObjects` recursively by re-interpreting the object +/// Functions for working with [ffi::PyObject]s recursively by re-interpreting the object /// as being an instance of the most derived class through each base class until /// the `BaseNativeType` is reached. /// @@ -86,8 +86,8 @@ pub trait PyObjectRecursiveOperations { /// Cleanup then free the memory for `obj`. /// /// # Safety - /// - slf must be a valid pointer to an instance of a T or a subclass. - /// - slf must not be used after this call (as it will be freed). + /// - `obj` must be a valid pointer to an instance of a `T` or a subclass. + /// - `obj` must not be used after this call (as it will be freed). unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject); } @@ -154,8 +154,8 @@ impl PyObjectRecursiveOperations /// [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) /// /// # Safety - /// - obj must be a valid pointer to an instance of the type at `type_ptr` or a subclass. - /// - obj must not be used after this call (as it will be freed). + /// - `obj` must be a valid pointer to an instance of the type at `type_ptr` or a subclass. + /// - `obj` must not be used after this call (as it will be freed). unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { // the `BaseNativeType` of the object let type_ptr = ::type_object_raw(py); @@ -208,7 +208,7 @@ impl PyObjectRecursiveOperations } } -/// Utilities for working with `PyObject` objects that utilise [PEP 697](https://peps.python.org/pep-0697/). +/// Utilities for working with [ffi::PyObject] objects that utilise [PEP 697](https://peps.python.org/pep-0697/). #[doc(hidden)] pub(crate) mod opaque_layout { #[cfg(Py_3_12)] @@ -255,7 +255,7 @@ pub(crate) mod opaque_layout { } } -/// Utilities for working with `PyObject` objects that utilise the standard layout for python extensions, +/// Utilities for working with [ffi::PyObject] objects that utilise the standard layout for python extensions, /// where the base class is placed at the beginning of a `repr(C)` struct. #[doc(hidden)] pub(crate) mod static_layout { @@ -266,7 +266,7 @@ pub(crate) mod static_layout { use super::PyClassObjectContents; - // The layout of a `PyObject` that uses the static layout + // The layout of a [ffi::PyObject] that uses the static layout #[repr(C)] pub struct PyStaticClassLayout { pub(crate) ob_base: ::StaticLayout, @@ -275,7 +275,7 @@ pub(crate) mod static_layout { unsafe impl PyLayout for PyStaticClassLayout {} - /// Base layout of PyClassObject with a known sized base type. + /// Base layout of [PyClassObject] with a known sized base type. /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. #[doc(hidden)] #[repr(C)] @@ -1698,8 +1698,8 @@ mod opaque_fail_tests { fn __init__(&mut self, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) {} } - /// PyType uses the opaque layout. Explicitly using `#[pyclass(opaque)]` can be caught at compile time - /// but it is possible to create a pyclass that uses the opaque layout by extending an opaque native type. + /// PyType uses the opaque layout. While explicitly using `#[pyclass(opaque)]` can be caught at compile time, + /// it is also possible to create a pyclass that uses the opaque layout by extending an opaque native type. #[test] #[should_panic( expected = "The opaque object layout (used by pyo3::pycell::layout::opaque_fail_tests::Metaclass) is not supported until python 3.12" diff --git a/src/type_object.rs b/src/type_object.rs index 13b7c3c732e..eacf9e04cf0 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -6,7 +6,7 @@ use crate::types::{PyAny, PyType}; use crate::{ffi, Bound, Python}; /// `T: PyNativeType` represents that `T` is a struct representing a 'native python class'. -/// a 'native class' is a wrapper around a `ffi::PyTypeObject` that is defined by the python +/// a 'native class' is a wrapper around a [ffi::PyTypeObject] that is defined by the python /// API such as `PyDict` for `dict`. /// /// This trait is intended to be used internally. From 8ad4da5dda939fbdb50084d333e04752c2b0dc18 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 25 Nov 2024 22:44:55 +0000 Subject: [PATCH 41/41] some polishing --- pyo3-macros-backend/src/pyclass.rs | 1 - pyo3-macros-backend/src/pymethod.rs | 2 -- src/impl_/pyclass.rs | 11 +++++--- src/pycell/borrow_checker.rs | 9 ------- src/pycell/layout.rs | 24 +++++++++-------- src/types/typeobject.rs | 2 ++ tests/test_class_init.rs | 42 +++-------------------------- tests/ui/invalid_opaque.stderr | 9 ------- 8 files changed, 27 insertions(+), 73 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 042b6ebb368..d5a949bd89c 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1827,7 +1827,6 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre let opaque = if attr.options.opaque.is_some() { quote! { - #[cfg(Py_3_12)] const OPAQUE: bool = true; #[cfg(not(Py_3_12))] diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 69e8749d934..fda1b360e40 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -405,8 +405,6 @@ pub fn impl_py_method_def_new( fn impl_init_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { let Ctx { pyo3_path, .. } = ctx; - // HACK: __init__ proto slot must always use varargs calling convention, so change the spec. - // Probably indicates there's a refactoring opportunity somewhere. spec.convention = CallingConvention::Varargs; let wrapper_ident = syn::Ident::new("__pymethod___init____", Span::call_site()); diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 965871d33de..468ddd8866c 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -211,14 +211,14 @@ pub trait PyClassImpl: Sized + 'static { fn items_iter() -> PyClassItemsIter; - /// Used to provide the __dictoffset__ slot + /// Used to provide the `__dictoffset__` slot /// (equivalent to [tp_dictoffset](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dictoffset)) #[inline] fn dict_offset() -> Option { None } - /// Used to provide the __weaklistoffset__ slot + /// Used to provide the `__weaklistoffset__` slot /// (equivalent to [tp_weaklistoffset](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_weaklistoffset) #[inline] fn weaklist_offset() -> Option { @@ -1147,10 +1147,15 @@ impl PyClassThreadChecker for ThreadCheckerImpl { )] pub trait PyClassBaseType: Sized { /// A struct that describes the memory layout of a `ffi:PyObject` with the type of `Self`. - /// Only valid when `::OPAQUE` is `false`. + /// Only valid when `::OPAQUE == false`. type StaticLayout: PyLayout; + /// The nearest ancestor in the inheritance tree that is a native type (not a `#[pyclass]` annotated struct). type BaseNativeType: PyTypeInfo; + /// The implementation for recursive operations that walk the inheritance tree back to the `BaseNativeType`. + /// (two implementations: one for native type, one for pyclass) type RecursiveOperations: PyObjectRecursiveOperations; + /// The implementation for constructing new a new `ffi::PyObject` of this type. + /// (two implementations: one for native type, one for pyclass) type Initializer: PyObjectInit; type PyClassMutability: PyClassMutability; } diff --git a/src/pycell/borrow_checker.rs b/src/pycell/borrow_checker.rs index 5569129a652..045b8c8b931 100644 --- a/src/pycell/borrow_checker.rs +++ b/src/pycell/borrow_checker.rs @@ -260,15 +260,6 @@ mod tests { #[pyclass(crate = "crate", extends = ImmutableChildOfImmutableBase, frozen)] struct ImmutableChildOfImmutableChildOfImmutableBase; - #[pyclass(crate = "crate", subclass)] - struct BaseWithData(#[allow(unused)] u64); - - #[pyclass(crate = "crate", extends = BaseWithData)] - struct ChildWithData(#[allow(unused)] u64); - - #[pyclass(crate = "crate", extends = BaseWithData)] - struct ChildWithoutData; - fn assert_mutable>() {} fn assert_immutable>() {} fn assert_mutable_with_mutable_ancestor< diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs index f651f0f6b13..1948f3a5d94 100644 --- a/src/pycell/layout.rs +++ b/src/pycell/layout.rs @@ -24,7 +24,7 @@ use crate::types::PyTypeMethods; use super::borrow_checker::PyClassMutability; use super::{ptr_from_ref, PyBorrowError}; -/// The data of a [ffi::PyObject] specifically relating to type `T`. +/// The layout of the region of a [ffi::PyObject] specifically relating to type `T`. /// /// In an inheritance hierarchy where `#[pyclass(extends=PyDict)] struct A;` and `#[pyclass(extends=A)] struct B;` /// a [ffi::PyObject] of type `B` has separate memory for [ffi::PyDictObject] (the base native type) and @@ -217,8 +217,12 @@ pub(crate) mod opaque_layout { use crate::ffi; use crate::{impl_::pyclass::PyClassImpl, PyTypeInfo}; + /// Obtain a pointer to the region of `obj` that relates to `T` + /// + /// # Safety + /// - `obj` must be a valid `ffi::PyObject` of type `T` or a subclass of `T` that uses the opaque layout #[cfg(Py_3_12)] - pub(crate) fn get_contents_ptr( + pub(crate) unsafe fn get_contents_ptr( obj: *mut ffi::PyObject, strategy: TypeObjectStrategy<'_>, ) -> *mut PyClassObjectContents { @@ -275,7 +279,7 @@ pub(crate) mod static_layout { unsafe impl PyLayout for PyStaticClassLayout {} - /// Base layout of [PyClassObject] with a known sized base type. + /// Layout of a native type `T` with a known size (not opaque) /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. #[doc(hidden)] #[repr(C)] @@ -291,7 +295,7 @@ pub(crate) mod static_layout { pub struct InvalidStaticLayout; /// This is valid insofar as casting a `*mut ffi::PyObject` to `*mut InvalidStaticLayout` is valid - /// since nothing can actually be read by dereferencing. + /// since `InvalidStaticLayout` has no fields to read. unsafe impl PyLayout for InvalidStaticLayout {} } @@ -358,7 +362,7 @@ impl PyObjectLayout { } } else { let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); - // indicates `ob_base` has type InvalidBaseLayout + // indicates `ob_base` has type [static_layout::InvalidStaticLayout] debug_assert_ne!( offset_of!(static_layout::PyStaticClassLayout, contents), 0, @@ -382,7 +386,7 @@ impl PyObjectLayout { )) } - /// Obtain a pointer to the portion of `obj` containing the data for `T` + /// Obtain a pointer to the portion of `obj` containing the user data for `T` /// /// # Safety /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. @@ -394,7 +398,7 @@ impl PyObjectLayout { (*contents).value.get() } - /// Obtain a reference to the portion of `obj` containing the data for `T` + /// Obtain a reference to the portion of `obj` containing the user data for `T` /// /// # Safety /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. @@ -569,8 +573,8 @@ mod static_tests { #[repr(C)] struct ExpectedLayout { - // typically called `ob_base`. In C it is defined using the `PyObject_HEAD` macro - // https://docs.python.org/3/c-api/structures.html + /// typically called `ob_base`. In C it is defined using the `PyObject_HEAD` macro + /// [docs](https://docs.python.org/3/c-api/structures.html) native_base: ffi::PyObject, contents: PyClassObjectContents, } @@ -622,8 +626,6 @@ mod static_tests { #[repr(C)] struct ExpectedLayout { - // typically called `ob_base`. In C it is defined using the `PyObject_HEAD` macro - // https://docs.python.org/3/c-api/structures.html native_base: ffi::PyObject, contents: PyClassObjectContents, } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index faa792eb80f..020c90126fb 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -26,6 +26,8 @@ pyobject_native_type_core!( pyobject_native_type_object_methods!(PyType, #global=ffi::PyType_Type); impl crate::impl_::pyclass::PyClassBaseType for PyType { + /// [ffi::PyType_Type] has a variable size and private fields even when using the unlimited API, it therefore + /// cannot be used with the static layout. Attempts to do so will panic when accessed. type StaticLayout = crate::impl_::pycell::InvalidStaticLayout; type BaseNativeType = PyType; type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; diff --git a/tests/test_class_init.rs b/tests/test_class_init.rs index 658fd120f39..dfee17f99f0 100644 --- a/tests/test_class_init.rs +++ b/tests/test_class_init.rs @@ -250,45 +250,17 @@ fn subclass_init() { } #[pyclass(extends=SuperClass)] -struct SubClass { - #[pyo3(get)] - rust_subclass_new: bool, - #[pyo3(get)] - rust_subclass_default: bool, - #[pyo3(get)] - rust_subclass_init: bool, -} - -impl Default for SubClass { - fn default() -> Self { - Self { - rust_subclass_new: false, - rust_subclass_default: true, - rust_subclass_init: false, - } - } -} +#[derive(Default)] +struct SubClass {} #[pymethods] impl SubClass { #[new] fn new() -> (Self, SuperClass) { - ( - SubClass { - rust_subclass_new: true, - rust_subclass_default: false, - rust_subclass_init: false, - }, - SuperClass::new(), - ) + (SubClass {}, SuperClass::new()) } - fn __init__(&mut self) { - assert!(!self.rust_subclass_new); - assert!(self.rust_subclass_default); - assert!(!self.rust_subclass_init); - self.rust_subclass_init = true; - } + fn __init__(&mut self) {} } #[test] @@ -301,12 +273,6 @@ fn subclass_pyclass_init() { let source = pyo3_ffi::c_str!(pyo3::indoc::indoc!( r#" c = SubClass() - assert c.rust_new is True - assert c.rust_default is False - assert c.rust_init is False - assert c.rust_subclass_new is False # overridden by calling __init__ - assert c.rust_subclass_default is True - assert c.rust_subclass_init is True "# )); let globals = PyModule::import(py, "__main__").unwrap().dict(); diff --git a/tests/ui/invalid_opaque.stderr b/tests/ui/invalid_opaque.stderr index 96b16507365..9ca79b46f72 100644 --- a/tests/ui/invalid_opaque.stderr +++ b/tests/ui/invalid_opaque.stderr @@ -5,12 +5,3 @@ error: #[pyclass(opaque)] requires python 3.12 or later | ^^^^^^^^^^^^^^^^^^ | = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) - -error[E0046]: not all trait items implemented, missing: `OPAQUE` - --> tests/ui/invalid_opaque.rs:3:1 - | -3 | #[pyclass(opaque)] - | ^^^^^^^^^^^^^^^^^^ missing `OPAQUE` in implementation - | - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) - = help: implement the missing item: `const OPAQUE: bool = false;`