From 89d4ae1dbf69b4c3d49d81cc752fc7ddbdae2753 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Thu, 19 Jan 2023 08:51:44 +0000
Subject: [PATCH 01/31] rename `wrap_pyfunction` impl to `wrap_pyfunction_impl`

---
 pyo3-macros-backend/src/module.rs | 2 +-
 src/impl_/pyfunction.rs           | 2 +-
 src/macros.rs                     | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs
index 236aee8e04b..cb2fda00c0f 100644
--- a/pyo3-macros-backend/src/module.rs
+++ b/pyo3-macros-backend/src/module.rs
@@ -122,7 +122,7 @@ pub fn process_functions_in_module(
                 let name = &func.sig.ident;
                 let statements: Vec<syn::Stmt> = syn::parse_quote! {
                     #wrapped_function
-                    #module_name.add_function(#krate::impl_::pyfunction::wrap_pyfunction(&#name::DEF, #module_name)?)?;
+                    #module_name.add_function(#krate::impl_::pyfunction::wrap_pyfunction_impl(&#name::DEF, #module_name)?)?;
                 };
                 stmts.extend(statements);
             }
diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs
index 2764c0689aa..95d8350d270 100644
--- a/src/impl_/pyfunction.rs
+++ b/src/impl_/pyfunction.rs
@@ -2,7 +2,7 @@ use crate::{derive_utils::PyFunctionArguments, types::PyCFunction, PyResult};
 
 pub use crate::impl_::pymethods::PyMethodDef;
 
-pub fn wrap_pyfunction<'a>(
+pub fn wrap_pyfunction_impl<'a>(
     method_def: &PyMethodDef,
     py_or_module: impl Into<PyFunctionArguments<'a>>,
 ) -> PyResult<&'a PyCFunction> {
diff --git a/src/macros.rs b/src/macros.rs
index caf72e1fbcf..07c87056209 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -125,12 +125,12 @@ macro_rules! wrap_pyfunction {
     ($function:path) => {
         &|py_or_module| {
             use $function as wrapped_pyfunction;
-            $crate::impl_::pyfunction::wrap_pyfunction(&wrapped_pyfunction::DEF, py_or_module)
+            $crate::impl_::pyfunction::wrap_pyfunction_impl(&wrapped_pyfunction::DEF, py_or_module)
         }
     };
     ($function:path, $py_or_module:expr) => {{
         use $function as wrapped_pyfunction;
-        $crate::impl_::pyfunction::wrap_pyfunction(&wrapped_pyfunction::DEF, $py_or_module)
+        $crate::impl_::pyfunction::wrap_pyfunction_impl(&wrapped_pyfunction::DEF, $py_or_module)
     }};
 }
 

From 586fed2c4b6b6d7c1147cd0bb7378216cb802b29 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Thu, 19 Jan 2023 07:45:05 +0000
Subject: [PATCH 02/31] send errors in `__releasebuffer__` to
 `sys.unraisablehook`

---
 guide/src/class/protocols.md  |   3 +-
 newsfragments/2886.fixed.md   |   1 +
 src/callback.rs               |   4 -
 src/impl_/pyclass.rs          |   7 +-
 src/impl_/trampoline.rs       |  59 ++++++++++-
 tests/test_buffer_protocol.rs | 181 ++++++++++++++++++++++++----------
 6 files changed, 186 insertions(+), 69 deletions(-)
 create mode 100644 newsfragments/2886.fixed.md

diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md
index c269eb3ade2..5f22c157902 100644
--- a/guide/src/class/protocols.md
+++ b/guide/src/class/protocols.md
@@ -362,7 +362,8 @@ Coercions:
 ### Buffer objects
 
   - `__getbuffer__(<self>, *mut ffi::Py_buffer, flags) -> ()`
-  - `__releasebuffer__(<self>, *mut ffi::Py_buffer)` (no return value, not even `PyResult`)
+  - `__releasebuffer__(<self>, *mut ffi::Py_buffer) -> ()`
+    Errors returned from `__releasebuffer__` will be sent to `sys.unraiseablehook`. It is strongly advised to never return an error from `__releasebuffer__`, and if it really is necessary, to make best effort to perform any required freeing operations before returning. `__releasebuffer__` will not be called a second time; anything not freed will be leaked.
 
 ### Garbage Collector Integration
 
diff --git a/newsfragments/2886.fixed.md b/newsfragments/2886.fixed.md
new file mode 100644
index 00000000000..cfe192d4ef0
--- /dev/null
+++ b/newsfragments/2886.fixed.md
@@ -0,0 +1 @@
+Send errors returned by `__releasebuffer__` to `sys.unraisablehook` rather than causing `SystemError`.
diff --git a/src/callback.rs b/src/callback.rs
index 588639bbd67..6d59253730e 100644
--- a/src/callback.rs
+++ b/src/callback.rs
@@ -28,10 +28,6 @@ impl PyCallbackOutput for ffi::Py_ssize_t {
     const ERR_VALUE: Self = -1;
 }
 
-impl PyCallbackOutput for () {
-    const ERR_VALUE: Self = ();
-}
-
 /// Convert the result of callback function into the appropriate return value.
 pub trait IntoPyCallbackOutput<Target> {
     fn convert(self, py: Python<'_>) -> PyResult<Target>;
diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs
index b7d400112ff..6d1a93732f4 100644
--- a/src/impl_/pyclass.rs
+++ b/src/impl_/pyclass.rs
@@ -931,13 +931,8 @@ pub(crate) unsafe extern "C" fn tp_dealloc<T: PyClass>(obj: *mut ffi::PyObject)
     /// A wrapper because PyCellLayout::tp_dealloc currently takes the py argument last
     /// (which is different to the rest of the trampolines which take py first)
     #[inline]
-    #[allow(clippy::unnecessary_wraps)]
-    unsafe fn trampoline_dealloc_wrapper<T: PyClass>(
-        py: Python<'_>,
-        slf: *mut ffi::PyObject,
-    ) -> PyResult<()> {
+    unsafe fn trampoline_dealloc_wrapper<T: PyClass>(py: Python<'_>, slf: *mut ffi::PyObject) {
         T::Layout::tp_dealloc(slf, py);
-        Ok(())
     }
     // TODO change argument order in PyCellLayout::tp_dealloc so this wrapper isn't needed.
     crate::impl_::trampoline::dealloc(obj, trampoline_dealloc_wrapper::<T>)
diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs
index 44227fa78fe..c75a68beacd 100644
--- a/src/impl_/trampoline.rs
+++ b/src/impl_/trampoline.rs
@@ -140,15 +140,39 @@ trampolines!(
     ) -> *mut ffi::PyObject;
 
     pub fn unaryfunc(slf: *mut ffi::PyObject) -> *mut ffi::PyObject;
-
-    pub fn dealloc(slf: *mut ffi::PyObject) -> ();
 );
 
 #[cfg(any(not(Py_LIMITED_API), Py_3_11))]
-trampolines! {
+trampoline! {
     pub fn getbufferproc(slf: *mut ffi::PyObject, buf: *mut ffi::Py_buffer, flags: c_int) -> c_int;
+}
+
+#[cfg(any(not(Py_LIMITED_API), Py_3_11))]
+#[inline]
+pub unsafe fn releasebufferproc(
+    slf: *mut ffi::PyObject,
+    buf: *mut ffi::Py_buffer,
+    f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject, *mut ffi::Py_buffer) -> PyResult<()>,
+) {
+    trampoline_inner_unraisable(|py| f(py, slf, buf), slf)
+}
 
-    pub fn releasebufferproc(slf: *mut ffi::PyObject, buf: *mut ffi::Py_buffer) -> ();
+#[inline]
+pub(crate) unsafe fn dealloc(
+    slf: *mut ffi::PyObject,
+    f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject) -> (),
+) {
+    // After calling tp_dealloc the object is no longer valid,
+    // so pass null_mut() to the context.
+    //
+    // (Note that we don't allow the implementation `f` to fail.)
+    trampoline_inner_unraisable(
+        |py| {
+            f(py, slf);
+            Ok(())
+        },
+        std::ptr::null_mut(),
+    )
 }
 
 // Ipowfunc is a unique case where PyO3 has its own type
@@ -201,3 +225,30 @@ where
     py_err.restore(py);
     R::ERR_VALUE
 }
+
+/// Implementation of trampoline for functions which can't return an error.
+///
+/// Panics during execution are trapped so that they don't propagate through any
+/// outer FFI boundary.
+///
+/// Exceptions produced are sent to `sys.unraisablehook`.
+///
+/// # Safety
+///
+/// ctx must be either a valid ffi::PyObject or NULL
+#[inline]
+unsafe fn trampoline_inner_unraisable<F>(body: F, ctx: *mut ffi::PyObject)
+where
+    F: for<'py> FnOnce(Python<'py>) -> PyResult<()> + UnwindSafe,
+{
+    let trap = PanicTrap::new("uncaught panic at ffi boundary");
+    let pool = GILPool::new();
+    let py = pool.python();
+    if let Err(py_err) = panic::catch_unwind(move || body(py))
+        .unwrap_or_else(|payload| Err(PanicException::from_panic_payload(payload)))
+    {
+        py_err.restore(py);
+        ffi::PyErr_WriteUnraisable(ctx);
+    }
+    trap.disarm();
+}
diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs
index c7d2a399a01..c3843909c8f 100644
--- a/tests/test_buffer_protocol.rs
+++ b/tests/test_buffer_protocol.rs
@@ -24,49 +24,11 @@ struct TestBufferClass {
 #[pymethods]
 impl TestBufferClass {
     unsafe fn __getbuffer__(
-        mut slf: PyRefMut<'_, Self>,
+        slf: &PyCell<Self>,
         view: *mut ffi::Py_buffer,
         flags: c_int,
     ) -> PyResult<()> {
-        if view.is_null() {
-            return Err(PyBufferError::new_err("View is null"));
-        }
-
-        if (flags & ffi::PyBUF_WRITABLE) == ffi::PyBUF_WRITABLE {
-            return Err(PyBufferError::new_err("Object is not writable"));
-        }
-
-        (*view).obj = ffi::_Py_NewRef(slf.as_ptr());
-
-        (*view).buf = slf.vec.as_mut_ptr() as *mut c_void;
-        (*view).len = slf.vec.len() as isize;
-        (*view).readonly = 1;
-        (*view).itemsize = 1;
-
-        (*view).format = if (flags & ffi::PyBUF_FORMAT) == ffi::PyBUF_FORMAT {
-            let msg = CString::new("B").unwrap();
-            msg.into_raw()
-        } else {
-            ptr::null_mut()
-        };
-
-        (*view).ndim = 1;
-        (*view).shape = if (flags & ffi::PyBUF_ND) == ffi::PyBUF_ND {
-            &mut (*view).len
-        } else {
-            ptr::null_mut()
-        };
-
-        (*view).strides = if (flags & ffi::PyBUF_STRIDES) == ffi::PyBUF_STRIDES {
-            &mut (*view).itemsize
-        } else {
-            ptr::null_mut()
-        };
-
-        (*view).suboffsets = ptr::null_mut();
-        (*view).internal = ptr::null_mut();
-
-        Ok(())
+        fill_view_from_readonly_data(view, flags, &slf.borrow().vec, slf)
     }
 
     unsafe fn __releasebuffer__(&self, view: *mut ffi::Py_buffer) {
@@ -86,20 +48,18 @@ impl Drop for TestBufferClass {
 fn test_buffer() {
     let drop_called = Arc::new(AtomicBool::new(false));
 
-    {
-        Python::with_gil(|py| {
-            let instance = Py::new(
-                py,
-                TestBufferClass {
-                    vec: vec![b' ', b'2', b'3'],
-                    drop_called: drop_called.clone(),
-                },
-            )
-            .unwrap();
-            let env = [("ob", instance)].into_py_dict(py);
-            py_assert!(py, *env, "bytes(ob) == b' 23'");
-        });
-    }
+    Python::with_gil(|py| {
+        let instance = Py::new(
+            py,
+            TestBufferClass {
+                vec: vec![b' ', b'2', b'3'],
+                drop_called: drop_called.clone(),
+            },
+        )
+        .unwrap();
+        let env = [("ob", instance)].into_py_dict(py);
+        py_assert!(py, *env, "bytes(ob) == b' 23'");
+    });
 
     assert!(drop_called.load(Ordering::Relaxed));
 }
@@ -132,3 +92,116 @@ fn test_buffer_referenced() {
 
     assert!(drop_called.load(Ordering::Relaxed));
 }
+
+#[test]
+#[cfg(Py_3_8)] // sys.unraisablehook not available until Python 3.8
+fn test_releasebuffer_unraisable_error() {
+    use pyo3::exceptions::PyValueError;
+
+    #[pyclass]
+    struct ReleaseBufferError {}
+
+    #[pymethods]
+    impl ReleaseBufferError {
+        unsafe fn __getbuffer__(
+            slf: &PyCell<Self>,
+            view: *mut ffi::Py_buffer,
+            flags: c_int,
+        ) -> PyResult<()> {
+            static BUF_BYTES: &[u8] = b"hello world";
+            fill_view_from_readonly_data(view, flags, BUF_BYTES, slf)
+        }
+
+        unsafe fn __releasebuffer__(&self, _view: *mut ffi::Py_buffer) -> PyResult<()> {
+            Err(PyValueError::new_err("oh dear"))
+        }
+    }
+
+    #[pyclass]
+    struct UnraisableCapture {
+        capture: Option<(PyErr, PyObject)>,
+    }
+
+    #[pymethods]
+    impl UnraisableCapture {
+        fn hook(&mut self, unraisable: &PyAny) {
+            let err = PyErr::from_value(unraisable.getattr("exc_value").unwrap());
+            let instance = unraisable.getattr("object").unwrap();
+            self.capture = Some((err, instance.into()));
+        }
+    }
+
+    Python::with_gil(|py| {
+        let sys = py.import("sys").unwrap();
+        let old_hook = sys.getattr("unraisablehook").unwrap();
+        let capture = Py::new(py, UnraisableCapture { capture: None }).unwrap();
+
+        sys.setattr("unraisablehook", capture.getattr(py, "hook").unwrap())
+            .unwrap();
+
+        let instance = Py::new(py, ReleaseBufferError {}).unwrap();
+        let env = [("ob", instance.clone())].into_py_dict(py);
+
+        assert!(capture.borrow(py).capture.is_none());
+
+        py_assert!(py, *env, "bytes(ob) == b'hello world'");
+
+        let (err, object) = capture.borrow_mut(py).capture.take().unwrap();
+        assert_eq!(err.to_string(), "ValueError: oh dear");
+        assert!(object.is(&instance));
+
+        sys.setattr("unraisablehook", old_hook).unwrap();
+    });
+}
+
+/// # Safety
+///
+/// `view` must be a valid pointer to ffi::Py_buffer, or null
+/// `data` must outlive the Python lifetime of `owner` (i.e. data must be owned by owner, or data
+/// must be static data)
+unsafe fn fill_view_from_readonly_data(
+    view: *mut ffi::Py_buffer,
+    flags: c_int,
+    data: &[u8],
+    owner: &PyAny,
+) -> PyResult<()> {
+    if view.is_null() {
+        return Err(PyBufferError::new_err("View is null"));
+    }
+
+    if (flags & ffi::PyBUF_WRITABLE) == ffi::PyBUF_WRITABLE {
+        return Err(PyBufferError::new_err("Object is not writable"));
+    }
+
+    (*view).obj = ffi::_Py_NewRef(owner.as_ptr());
+
+    (*view).buf = data.as_ptr() as *mut c_void;
+    (*view).len = data.len() as isize;
+    (*view).readonly = 1;
+    (*view).itemsize = 1;
+
+    (*view).format = if (flags & ffi::PyBUF_FORMAT) == ffi::PyBUF_FORMAT {
+        let msg = CString::new("B").unwrap();
+        msg.into_raw()
+    } else {
+        ptr::null_mut()
+    };
+
+    (*view).ndim = 1;
+    (*view).shape = if (flags & ffi::PyBUF_ND) == ffi::PyBUF_ND {
+        &mut (*view).len
+    } else {
+        ptr::null_mut()
+    };
+
+    (*view).strides = if (flags & ffi::PyBUF_STRIDES) == ffi::PyBUF_STRIDES {
+        &mut (*view).itemsize
+    } else {
+        ptr::null_mut()
+    };
+
+    (*view).suboffsets = ptr::null_mut();
+    (*view).internal = ptr::null_mut();
+
+    Ok(())
+}

From 255d9bacce64c4a0712e75c1c4635d0a034d7e51 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Thu, 19 Jan 2023 21:05:04 +0000
Subject: [PATCH 03/31] tidy up implementation of pyclass `tp_dealloc`

---
 src/impl_/pyclass.rs | 9 +--------
 src/pycell.rs        | 8 ++++----
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs
index 6d1a93732f4..a2671748d74 100644
--- a/src/impl_/pyclass.rs
+++ b/src/impl_/pyclass.rs
@@ -928,14 +928,7 @@ impl<T: PyClass> PyClassBaseType for T {
 
 /// Implementation of tp_dealloc for all pyclasses
 pub(crate) unsafe extern "C" fn tp_dealloc<T: PyClass>(obj: *mut ffi::PyObject) {
-    /// A wrapper because PyCellLayout::tp_dealloc currently takes the py argument last
-    /// (which is different to the rest of the trampolines which take py first)
-    #[inline]
-    unsafe fn trampoline_dealloc_wrapper<T: PyClass>(py: Python<'_>, slf: *mut ffi::PyObject) {
-        T::Layout::tp_dealloc(slf, py);
-    }
-    // TODO change argument order in PyCellLayout::tp_dealloc so this wrapper isn't needed.
-    crate::impl_::trampoline::dealloc(obj, trampoline_dealloc_wrapper::<T>)
+    crate::impl_::trampoline::dealloc(obj, T::Layout::tp_dealloc)
 }
 
 pub(crate) unsafe extern "C" fn get_sequence_item_from_mapping(
diff --git a/src/pycell.rs b/src/pycell.rs
index 18942f448f3..0be39ac674d 100644
--- a/src/pycell.rs
+++ b/src/pycell.rs
@@ -854,7 +854,7 @@ pub trait PyCellLayout<T>: PyLayout<T> {
     /// # Safety
     /// - slf must be a valid pointer to an instance of a T or a subclass.
     /// - slf must not be used after this call (as it will be freed).
-    unsafe fn tp_dealloc(slf: *mut ffi::PyObject, py: Python<'_>);
+    unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject);
 }
 
 impl<T, U> PyCellLayout<T> for PyCellBase<U>
@@ -863,7 +863,7 @@ where
     T: PyTypeInfo,
 {
     fn ensure_threadsafe(&self) {}
-    unsafe fn tp_dealloc(slf: *mut ffi::PyObject, py: Python<'_>) {
+    unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) {
         // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free
         if T::type_object_raw(py) == &mut PyBaseObject_Type {
             return get_tp_free(ffi::Py_TYPE(slf))(slf as _);
@@ -892,13 +892,13 @@ where
         self.contents.thread_checker.ensure();
         self.ob_base.ensure_threadsafe();
     }
-    unsafe fn tp_dealloc(slf: *mut ffi::PyObject, py: Python<'_>) {
+    unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) {
         // Safety: Python only calls tp_dealloc when no references to the object remain.
         let cell = &mut *(slf as *mut PyCell<T>);
         ManuallyDrop::drop(&mut cell.contents.value);
         cell.contents.dict.clear_dict(py);
         cell.contents.weakref.clear_weakrefs(slf, py);
-        <T::BaseType as PyClassBaseType>::LayoutAsBase::tp_dealloc(slf, py)
+        <T::BaseType as PyClassBaseType>::LayoutAsBase::tp_dealloc(py, slf)
     }
 }
 

From 16d347d96a6dc14cee1ced1153f2dc4b381c4aa0 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 20 Jan 2023 07:19:13 +0000
Subject: [PATCH 04/31] remove benchmarks from deprecated_pyfunctions

---
 pytests/tests/test_deprecated_pyfunctions.py | 42 +++-----------------
 1 file changed, 6 insertions(+), 36 deletions(-)

diff --git a/pytests/tests/test_deprecated_pyfunctions.py b/pytests/tests/test_deprecated_pyfunctions.py
index ec44d6207e2..9fb0a16c9a5 100644
--- a/pytests/tests/test_deprecated_pyfunctions.py
+++ b/pytests/tests/test_deprecated_pyfunctions.py
@@ -5,87 +5,57 @@ def none_py():
     return None
 
 
-def test_none_py(benchmark):
-    benchmark(none_py)
-
-
-def test_none_rs(benchmark):
+def test_none_rs():
     rust = pyfunctions.none()
     py = none_py()
     assert rust == py
-    benchmark(pyfunctions.none)
 
 
 def simple_py(a, b="bar", *, c=None):
     return a, b, c
 
 
-def test_simple_py(benchmark):
-    benchmark(simple_py, 1, "foo", c={1: 2})
-
-
-def test_simple_rs(benchmark):
+def test_simple_rs():
     rust = pyfunctions.simple(1, "foo", c={1: 2})
     py = simple_py(1, "foo", c={1: 2})
     assert rust == py
-    benchmark(pyfunctions.simple, 1, "foo", c={1: 2})
 
 
 def simple_args_py(a, b="bar", *args, c=None):
     return a, b, args, c
 
 
-def test_simple_args_py(benchmark):
-    benchmark(simple_args_py, 1, "foo", 4, 5, 6, c={1: 2})
-
-
-def test_simple_args_rs(benchmark):
+def test_simple_args_rs():
     rust = pyfunctions.simple_args(1, "foo", 4, 5, 6, c={1: 2})
     py = simple_args_py(1, "foo", 4, 5, 6, c={1: 2})
     assert rust == py
-    benchmark(pyfunctions.simple_args, 1, "foo", 4, 5, 6, c={1: 2})
 
 
 def simple_kwargs_py(a, b="bar", c=None, **kwargs):
     return a, b, c, kwargs
 
 
-def test_simple_kwargs_py(benchmark):
-    benchmark(simple_kwargs_py, 1, "foo", c={1: 2}, bar=4, foo=10)
-
-
-def test_simple_kwargs_rs(benchmark):
+def test_simple_kwargs_rs():
     rust = pyfunctions.simple_kwargs(1, "foo", c={1: 2}, bar=4, foo=10)
     py = simple_kwargs_py(1, "foo", c={1: 2}, bar=4, foo=10)
     assert rust == py
-    benchmark(pyfunctions.simple_kwargs, 1, "foo", c={1: 2}, bar=4, foo=10)
 
 
 def simple_args_kwargs_py(a, b="bar", *args, c=None, **kwargs):
     return (a, b, args, c, kwargs)
 
 
-def test_simple_args_kwargs_py(benchmark):
-    benchmark(simple_args_kwargs_py, 1, "foo", "baz", bar=4, foo=10)
-
-
-def test_simple_args_kwargs_rs(benchmark):
+def test_simple_args_kwargs_rs():
     rust = pyfunctions.simple_args_kwargs(1, "foo", "baz", bar=4, foo=10)
     py = simple_args_kwargs_py(1, "foo", "baz", bar=4, foo=10)
     assert rust == py
-    benchmark(pyfunctions.simple_args_kwargs, 1, "foo", "baz", bar=4, foo=10)
 
 
 def args_kwargs_py(*args, **kwargs):
     return (args, kwargs)
 
 
-def test_args_kwargs_py(benchmark):
-    benchmark(args_kwargs_py, 1, "foo", {1: 2}, bar=4, foo=10)
-
-
-def test_args_kwargs_rs(benchmark):
+def test_args_kwargs_rs():
     rust = pyfunctions.args_kwargs(1, "foo", {1: 2}, bar=4, foo=10)
     py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10)
     assert rust == py
-    benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, a=4, foo=10)

From 03628557732795cb42b94eec26fb81d8b599a9d0 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 20 Jan 2023 07:24:05 +0000
Subject: [PATCH 05/31] make rust benchmarks more similar to the Python ones

---
 pytests/src/pyfunctions.rs        | 47 ++++++++++++++++++-------------
 pytests/tests/test_pyfunctions.py | 26 +++++++----------
 2 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs
index 855f23afd5b..1eef970430e 100644
--- a/pytests/src/pyfunctions.rs
+++ b/pytests/src/pyfunctions.rs
@@ -4,43 +4,52 @@ use pyo3::types::{PyDict, PyTuple};
 #[pyfunction(signature = ())]
 fn none() {}
 
-#[pyfunction(signature = (a, b = "bar", *, c = None))]
-fn simple<'a>(a: i32, b: &'a str, c: Option<&'a PyDict>) -> (i32, &'a str, Option<&'a PyDict>) {
+#[pyfunction(signature = (a, b = None, *, c = None))]
+fn simple<'a>(
+    a: &'a PyAny,
+    b: Option<&'a PyAny>,
+    c: Option<&'a PyAny>,
+) -> (&'a PyAny, Option<&'a PyAny>, Option<&'a PyAny>) {
     (a, b, c)
 }
 
-#[pyfunction(signature = (a, b = "bar", *args, c = None))]
+#[pyfunction(signature = (a, b = None, *args, c = None))]
 fn simple_args<'a>(
-    a: i32,
-    b: &'a str,
+    a: &'a PyAny,
+    b: Option<&'a PyAny>,
     args: &'a PyTuple,
-    c: Option<&'a PyDict>,
-) -> (i32, &'a str, &'a PyTuple, Option<&'a PyDict>) {
+    c: Option<&'a PyAny>,
+) -> (&'a PyAny, Option<&'a PyAny>, &'a PyTuple, Option<&'a PyAny>) {
     (a, b, args, c)
 }
 
-#[pyfunction(signature = (a, b = "bar", c = None, **kwargs))]
+#[pyfunction(signature = (a, b = None, c = None, **kwargs))]
 fn simple_kwargs<'a>(
-    a: i32,
-    b: &'a str,
-    c: Option<&'a PyDict>,
+    a: &'a PyAny,
+    b: Option<&'a PyAny>,
+    c: Option<&'a PyAny>,
     kwargs: Option<&'a PyDict>,
-) -> (i32, &'a str, Option<&'a PyDict>, Option<&'a PyDict>) {
+) -> (
+    &'a PyAny,
+    Option<&'a PyAny>,
+    Option<&'a PyAny>,
+    Option<&'a PyDict>,
+) {
     (a, b, c, kwargs)
 }
 
-#[pyfunction(signature = (a, b = "bar", *args, c = None, **kwargs))]
+#[pyfunction(signature = (a, b = None, *args, c = None, **kwargs))]
 fn simple_args_kwargs<'a>(
-    a: i32,
-    b: &'a str,
+    a: &'a PyAny,
+    b: Option<&'a PyAny>,
     args: &'a PyTuple,
-    c: Option<&'a PyDict>,
+    c: Option<&'a PyAny>,
     kwargs: Option<&'a PyDict>,
 ) -> (
-    i32,
-    &'a str,
+    &'a PyAny,
+    Option<&'a PyAny>,
     &'a PyTuple,
-    Option<&'a PyDict>,
+    Option<&'a PyAny>,
     Option<&'a PyDict>,
 ) {
     (a, b, args, c, kwargs)
diff --git a/pytests/tests/test_pyfunctions.py b/pytests/tests/test_pyfunctions.py
index 1dda25285b6..c6fb448248b 100644
--- a/pytests/tests/test_pyfunctions.py
+++ b/pytests/tests/test_pyfunctions.py
@@ -10,13 +10,12 @@ def test_none_py(benchmark):
 
 
 def test_none_rs(benchmark):
-    rust = pyfunctions.none()
+    rust = benchmark(pyfunctions.none)
     py = none_py()
     assert rust == py
-    benchmark(pyfunctions.none)
 
 
-def simple_py(a, b="bar", *, c=None):
+def simple_py(a, b=None, *, c=None):
     return a, b, c
 
 
@@ -25,13 +24,12 @@ def test_simple_py(benchmark):
 
 
 def test_simple_rs(benchmark):
-    rust = pyfunctions.simple(1, "foo", c={1: 2})
+    rust = benchmark(pyfunctions.simple, 1, "foo", c={1: 2})
     py = simple_py(1, "foo", c={1: 2})
     assert rust == py
-    benchmark(pyfunctions.simple, 1, "foo", c={1: 2})
 
 
-def simple_args_py(a, b="bar", *args, c=None):
+def simple_args_py(a, b=None, *args, c=None):
     return a, b, args, c
 
 
@@ -40,13 +38,12 @@ def test_simple_args_py(benchmark):
 
 
 def test_simple_args_rs(benchmark):
-    rust = pyfunctions.simple_args(1, "foo", 4, 5, 6, c={1: 2})
+    rust = benchmark(pyfunctions.simple_args, 1, "foo", 4, 5, 6, c={1: 2})
     py = simple_args_py(1, "foo", 4, 5, 6, c={1: 2})
     assert rust == py
-    benchmark(pyfunctions.simple_args, 1, "foo", 4, 5, 6, c={1: 2})
 
 
-def simple_kwargs_py(a, b="bar", c=None, **kwargs):
+def simple_kwargs_py(a, b=None, c=None, **kwargs):
     return a, b, c, kwargs
 
 
@@ -55,13 +52,12 @@ def test_simple_kwargs_py(benchmark):
 
 
 def test_simple_kwargs_rs(benchmark):
-    rust = pyfunctions.simple_kwargs(1, "foo", c={1: 2}, bar=4, foo=10)
+    rust = benchmark(pyfunctions.simple_kwargs, 1, "foo", c={1: 2}, bar=4, foo=10)
     py = simple_kwargs_py(1, "foo", c={1: 2}, bar=4, foo=10)
     assert rust == py
-    benchmark(pyfunctions.simple_kwargs, 1, "foo", c={1: 2}, bar=4, foo=10)
 
 
-def simple_args_kwargs_py(a, b="bar", *args, c=None, **kwargs):
+def simple_args_kwargs_py(a, b=None, *args, c=None, **kwargs):
     return (a, b, args, c, kwargs)
 
 
@@ -70,10 +66,9 @@ def test_simple_args_kwargs_py(benchmark):
 
 
 def test_simple_args_kwargs_rs(benchmark):
-    rust = pyfunctions.simple_args_kwargs(1, "foo", "baz", bar=4, foo=10)
+    rust = benchmark(pyfunctions.simple_args_kwargs, 1, "foo", "baz", bar=4, foo=10)
     py = simple_args_kwargs_py(1, "foo", "baz", bar=4, foo=10)
     assert rust == py
-    benchmark(pyfunctions.simple_args_kwargs, 1, "foo", "baz", bar=4, foo=10)
 
 
 def args_kwargs_py(*args, **kwargs):
@@ -85,7 +80,6 @@ def test_args_kwargs_py(benchmark):
 
 
 def test_args_kwargs_rs(benchmark):
-    rust = pyfunctions.args_kwargs(1, "foo", {1: 2}, bar=4, foo=10)
+    rust = benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, bar=4, foo=10)
     py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10)
     assert rust == py
-    benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, a=4, foo=10)

From e83803e81303eb82c3fb9171915258844d03de87 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 20 Jan 2023 08:21:57 +0000
Subject: [PATCH 06/31] add link on how to obtain GIL to guide

---
 guide/src/types.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/guide/src/types.md b/guide/src/types.md
index 4ecc00aff36..7af771feb24 100644
--- a/guide/src/types.md
+++ b/guide/src/types.md
@@ -6,7 +6,7 @@ an overview of their intended meaning, with examples when each type is best
 used.
 
 
-## Mutability and Rust types
+## The Python GIL, mutability, and Rust types
 
 Since Python has no concept of ownership, and works solely with boxed objects,
 any Python object can be referenced any number of times, and mutation is allowed
@@ -43,6 +43,9 @@ an object's ownership has been passed to the Python interpreter, ensuring
 references is done at runtime using `PyCell`, a scheme very similar to
 `std::cell::RefCell`.
 
+### Accessing the Python GIL
+
+To get hold of a `Python<'py>` token to prove the GIL is held, consult [PyO3's documentation][obtaining-py].
 
 ## Object types
 
@@ -299,3 +302,4 @@ This trait marks structs that mirror native Python types, such as `PyList`.
 [PyAny]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html
 [PyList_append]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyList.html#method.append
 [RefCell]: https://doc.rust-lang.org/std/cell/struct.RefCell.html
+[obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token

From fe7b1eed046e2cbc5b8321941a5cadbc04e4b075 Mon Sep 17 00:00:00 2001
From: mejrs <59372212+mejrs@users.noreply.github.com>
Date: Tue, 24 Jan 2023 19:15:32 +0100
Subject: [PATCH 07/31] Update `downcast` documentation

---
 src/instance.rs  | 51 ++++++++++++++++++++++++++++++++++++++++++------
 src/types/any.rs | 36 +++++++++++++++++++++++++++++++---
 2 files changed, 78 insertions(+), 9 deletions(-)

diff --git a/src/instance.rs b/src/instance.rs
index b96c4cd5a42..9c473c84b74 100644
--- a/src/instance.rs
+++ b/src/instance.rs
@@ -991,10 +991,52 @@ impl<T> std::fmt::Debug for Py<T> {
 pub type PyObject = Py<PyAny>;
 
 impl PyObject {
-    /// Casts the PyObject to a concrete Python object type.
+    /// Downcast this `PyObject` to a concrete Python type or pyclass.
+    ///
+    /// Note that you can often avoid downcasting yourself by just specifying
+    /// the desired type in function or method signatures.
+    /// However, manual downcasting is sometimes necessary.
+    ///
+    /// For extracting a Rust-only type, see [`Py::extract`](struct.Py.html#method.extract).
+    ///
+    /// # Example: Downcasting to a specific Python object
+    ///
+    /// ```rust
+    /// use pyo3::prelude::*;
+    /// use pyo3::types::{PyDict, PyList};
+    ///
+    /// Python::with_gil(|py| {
+    ///     let any: PyObject = PyDict::new(py).into();
+    ///
+    ///     assert!(any.downcast::<PyDict>(py).is_ok());
+    ///     assert!(any.downcast::<PyList>(py).is_err());
+    /// });
+    /// ```
+    ///
+    /// # Example: Getting a reference to a pyclass
+    ///
+    /// This is useful if you want to mutate a `PyObject` that
+    /// might actually be a pyclass.
     ///
-    /// This can cast only to native Python types, not types implemented in Rust. For a more
-    /// flexible alternative, see [`Py::extract`](struct.Py.html#method.extract).
+    /// ```
+    /// # fn main() -> Result<(), pyo3::PyErr> {
+    /// use pyo3::prelude::*;
+    ///
+    /// #[pyclass]
+    /// struct Class {
+    ///     i: i32,
+    /// }
+    ///
+    /// Python::with_gil(|py| {
+    ///     let class: PyObject = Py::new(py, Class { i: 0 }).unwrap().into_py(py);
+    ///
+    ///     let class_cell: &PyCell<Class> = class.downcast(py)?;
+    ///
+    ///     class_cell.borrow_mut().i += 1;
+    ///     Ok(())
+    /// })
+    /// # }
+    /// ```
     #[inline]
     pub fn downcast<'p, T>(&'p self, py: Python<'p>) -> Result<&T, PyDowncastError<'_>>
     where
@@ -1005,9 +1047,6 @@ impl PyObject {
 
     /// Casts the PyObject to a concrete Python object type without checking validity.
     ///
-    /// This can cast only to native Python types, not types implemented in Rust. For a more
-    /// flexible alternative, see [`Py::extract`](struct.Py.html#method.extract).
-    ///
     /// # Safety
     ///
     /// Callers must ensure that the type is valid or risk type confusion.
diff --git a/src/types/any.rs b/src/types/any.rs
index 5abe97854b3..9a57fc17600 100644
--- a/src/types/any.rs
+++ b/src/types/any.rs
@@ -755,11 +755,15 @@ impl PyAny {
         self.downcast()
     }
 
-    /// Converts this `PyAny` to a concrete Python type.
+    /// Downcast this `PyAny` to a concrete Python type or pyclass.
     ///
-    /// This can cast only to native Python types, not types implemented in Rust.
+    /// Note that you can often avoid downcasting yourself by just specifying
+    /// the desired type in function or method signatures.
+    /// However, manual downcasting is sometimes necessary.
     ///
-    /// # Examples
+    /// For extracting a Rust-only type, see [`PyAny::extract`](struct.PyAny.html#method.extract).
+    ///
+    /// # Example: Downcasting to a specific Python object
     ///
     /// ```rust
     /// use pyo3::prelude::*;
@@ -769,10 +773,36 @@ impl PyAny {
     ///     let dict = PyDict::new(py);
     ///     assert!(dict.is_instance_of::<PyAny>().unwrap());
     ///     let any: &PyAny = dict.as_ref();
+    ///
     ///     assert!(any.downcast::<PyDict>().is_ok());
     ///     assert!(any.downcast::<PyList>().is_err());
     /// });
     /// ```
+    ///
+    /// # Example: Getting a reference to a pyclass
+    ///
+    /// This is useful if you want to mutate a `PyObject` that
+    /// might actually be a pyclass.
+    ///
+    /// ```
+    /// # fn main() -> Result<(), pyo3::PyErr> {
+    /// use pyo3::prelude::*;
+    ///
+    /// #[pyclass]
+    /// struct Class {
+    ///     i: i32,
+    /// }
+    ///
+    /// Python::with_gil(|py| {
+    ///     let class: &PyAny = Py::new(py, Class { i: 0 }).unwrap().into_ref(py);
+    ///
+    ///     let class_cell: &PyCell<Class> = class.downcast()?;
+    ///
+    ///     class_cell.borrow_mut().i += 1;
+    ///     Ok(())
+    /// })
+    /// # }
+    /// ```
     #[inline]
     pub fn downcast<'p, T>(&'p self) -> Result<&'p T, PyDowncastError<'_>>
     where

From e637b39abd88bc9629d69b24dfe4d94d645bf906 Mon Sep 17 00:00:00 2001
From: Sunyeop Lee <sunyeop97@gmail.com>
Date: Thu, 26 Jan 2023 21:54:27 +0900
Subject: [PATCH 08/31] Fix link for #2879

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce9fdd205e5..71123e40169 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,7 +48,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h
 - Accept any iterator in `PySet::new` and `PyFrozenSet::new`. [#2795](https://github.com/PyO3/pyo3/pull/2795)
 - Mixing `#[cfg(...)]` and `#[pyo3(...)]` attributes on `#[pyclass]` struct fields will now work. [#2796](https://github.com/PyO3/pyo3/pull/2796)
 - Re-enable `PyFunction` on when building for abi3 or PyPy. [#2838](https://github.com/PyO3/pyo3/pull/2838)
-- Improve `derive(FromPyObject)` to use `intern!` when applicable for `#[pyo3(item)]`. [#2838](https://github.com/PyO3/pyo3/pull/2838)
+- Improve `derive(FromPyObject)` to use `intern!` when applicable for `#[pyo3(item)]`. [#2879](https://github.com/PyO3/pyo3/pull/2879)
 
 ### Removed
 

From 2052269aeab23aaf0a503b0675396b1e3ff47e22 Mon Sep 17 00:00:00 2001
From: mejrs <59372212+mejrs@users.noreply.github.com>
Date: Thu, 26 Jan 2023 21:44:05 +0100
Subject: [PATCH 09/31] Update rust 1.67

---
 tests/ui/invalid_pymethod_receiver.stderr | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/tests/ui/invalid_pymethod_receiver.stderr b/tests/ui/invalid_pymethod_receiver.stderr
index d480a5c8080..83f29f548a0 100644
--- a/tests/ui/invalid_pymethod_receiver.stderr
+++ b/tests/ui/invalid_pymethod_receiver.stderr
@@ -5,14 +5,11 @@ error[E0277]: the trait bound `i32: From<&PyCell<MyClass>>` is not satisfied
   |                                           ^^^ the trait `From<&PyCell<MyClass>>` is not implemented for `i32`
   |
   = help: the following other types implement trait `From<T>`:
-            <f32 as From<i16>>
-            <f32 as From<i8>>
-            <f32 as From<u16>>
-            <f32 as From<u8>>
-            <f64 as From<f32>>
-            <f64 as From<i16>>
-            <f64 as From<i32>>
-            <f64 as From<i8>>
-          and $N others
+            <i32 as From<NonZeroI32>>
+            <i32 as From<bool>>
+            <i32 as From<i16>>
+            <i32 as From<i8>>
+            <i32 as From<u16>>
+            <i32 as From<u8>>
   = note: required for `&PyCell<MyClass>` to implement `Into<i32>`
   = note: required for `i32` to implement `TryFrom<&PyCell<MyClass>>`

From f38841a8e2443ffd11b04a906117828fa3595e81 Mon Sep 17 00:00:00 2001
From: Nate Kent <nate@nkent.net>
Date: Thu, 26 Jan 2023 11:21:20 -0800
Subject: [PATCH 10/31] Check to see if object is `None` before traversing

Closes #2915

When using the C API directly, the intended way to call `visitproc` is
via the `Py_VISIT` macro, which checks to see that the provided pointer
is not null before passing it along to `visitproc`. Because PyO3 isn't
using the macro, it needs to manually check that the pointer isn't null.
Without this check, calling `visit.call(&obj)` where `let obj = None;`
will segfault.
---
 newsfragments/2921.fixed.md |  1 +
 src/pyclass/gc.rs           | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)
 create mode 100644 newsfragments/2921.fixed.md

diff --git a/newsfragments/2921.fixed.md b/newsfragments/2921.fixed.md
new file mode 100644
index 00000000000..8cbffabb3ce
--- /dev/null
+++ b/newsfragments/2921.fixed.md
@@ -0,0 +1 @@
+Traversal visit calls to `Option<T: AsPyPointer>` no longer segfaults when `None`.
diff --git a/src/pyclass/gc.rs b/src/pyclass/gc.rs
index 8203af0789a..900027f7bfb 100644
--- a/src/pyclass/gc.rs
+++ b/src/pyclass/gc.rs
@@ -23,11 +23,16 @@ impl<'p> PyVisit<'p> {
     where
         T: AsPyPointer,
     {
-        let r = unsafe { (self.visit)(obj.as_ptr(), self.arg) };
-        if r == 0 {
-            Ok(())
+        let ptr = obj.as_ptr();
+        if !ptr.is_null() {
+            let r = unsafe { (self.visit)(ptr, self.arg) };
+            if r == 0 {
+                Ok(())
+            } else {
+                Err(PyTraverseError(r))
+            }
         } else {
-            Err(PyTraverseError(r))
+            Ok(())
         }
     }
 

From f11290d314c3b801300bdf001f7c33fd8fa473c4 Mon Sep 17 00:00:00 2001
From: Nate Kent <nate@nkent.net>
Date: Thu, 26 Jan 2023 16:25:18 -0800
Subject: [PATCH 11/31] Add additional unit test for GC traversal

Since we're just testing a bug during traversal, we don't actually have
to reap the object, we just have to make a reference cycle so that it's
traverse method is called.
---
 tests/test_gc.rs | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tests/test_gc.rs b/tests/test_gc.rs
index 28a0147ac03..8d612eb9c60 100644
--- a/tests/test_gc.rs
+++ b/tests/test_gc.rs
@@ -120,6 +120,44 @@ fn gc_integration() {
     });
 }
 
+#[pyclass]
+struct GcNullTraversal {
+    cycle: Option<Py<Self>>,
+    null: Option<Py<Self>>,
+}
+
+#[pymethods]
+impl GcNullTraversal {
+    fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.cycle)?;
+        visit.call(&self.null)?; // Should not segfault
+        Ok(())
+    }
+
+    fn __clear__(&mut self) {
+        self.cycle = None;
+        self.null = None;
+    }
+}
+
+#[test]
+fn gc_null_traversal() {
+    Python::with_gil(|py| {
+        let obj = Py::new(
+            py,
+            GcNullTraversal {
+                cycle: None,
+                null: None,
+            },
+        )
+        .unwrap();
+        obj.borrow_mut(py).cycle = Some(obj.clone_ref(py));
+
+        // the object doesn't have to be cleaned up, it just needs to be traversed.
+        py.run("import gc; gc.collect()", None, None).unwrap();
+    });
+}
+
 #[pyclass(subclass)]
 struct BaseClassWithDrop {
     data: Option<Arc<AtomicBool>>,

From 5667a095d61189c6bf95cd75965a077dc62c5382 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 27 Jan 2023 06:34:12 +0000
Subject: [PATCH 12/31] hygiene: fix `#[pymethods(crate = "...")]`

---
 newsfragments/2923.fixed.md   |  1 +
 pyo3-macros/src/lib.rs        | 15 ++++++++++++---
 src/test_hygiene/pymethods.rs |  8 ++++++++
 3 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 newsfragments/2923.fixed.md

diff --git a/newsfragments/2923.fixed.md b/newsfragments/2923.fixed.md
new file mode 100644
index 00000000000..d61f5b01e65
--- /dev/null
+++ b/newsfragments/2923.fixed.md
@@ -0,0 +1 @@
+Fix `#[pymethods(crate = "...")]` option being ignored.
diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs
index 730aaf65050..387934310b9 100644
--- a/pyo3-macros/src/lib.rs
+++ b/pyo3-macros/src/lib.rs
@@ -109,13 +109,13 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream {
 /// [10]: https://pyo3.rs/latest/class.html#method-arguments
 /// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set
 #[proc_macro_attribute]
-pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream {
+pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream {
     let methods_type = if cfg!(feature = "multiple-pymethods") {
         PyClassMethodsType::Inventory
     } else {
         PyClassMethodsType::Specialization
     };
-    pymethods_impl(input, methods_type)
+    pymethods_impl(attr, input, methods_type)
 }
 
 /// A proc macro used to expose Rust functions to Python.
@@ -191,8 +191,17 @@ fn pyclass_enum_impl(
     .into()
 }
 
-fn pymethods_impl(input: TokenStream, methods_type: PyClassMethodsType) -> TokenStream {
+fn pymethods_impl(
+    attr: TokenStream,
+    input: TokenStream,
+    methods_type: PyClassMethodsType,
+) -> TokenStream {
     let mut ast = parse_macro_input!(input as syn::ItemImpl);
+    // Apply all options as a #[pyo3] attribute on the ItemImpl
+    // e.g. #[pymethods(crate = "crate")] impl Foo { }
+    // -> #[pyo3(crate = "crate")] impl Foo { }
+    let attr: TokenStream2 = attr.into();
+    ast.attrs.push(syn::parse_quote!( #[pyo3(#attr)] ));
     let expanded = build_py_methods(&mut ast, methods_type).unwrap_or_compile_error();
 
     quote!(
diff --git a/src/test_hygiene/pymethods.rs b/src/test_hygiene/pymethods.rs
index eb0b12f73b8..ba0fcdff2b8 100644
--- a/src/test_hygiene/pymethods.rs
+++ b/src/test_hygiene/pymethods.rs
@@ -807,3 +807,11 @@ impl Dummy {
     // PyGcProtocol
     // Buffer protocol?
 }
+
+// Ensure that crate argument is also accepted inline
+
+#[crate::pyclass(crate = "crate")]
+struct Dummy2;
+
+#[crate::pymethods(crate = "crate")]
+impl Dummy2 {}

From e4a3d2893a2ab64c1d8c3ce2f4b1ac28646bb55c Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Tue, 24 Jan 2023 08:40:05 +0000
Subject: [PATCH 13/31] refactor docstring generation code

---
 pyo3-macros-backend/src/utils.rs | 117 +++++++++++++++++--------------
 1 file changed, 66 insertions(+), 51 deletions(-)

diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs
index ad20e917221..03a1d2a6bbf 100644
--- a/pyo3-macros-backend/src/utils.rs
+++ b/pyo3-macros-backend/src/utils.rs
@@ -1,9 +1,9 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, fmt::Write};
 
 // Copyright (c) 2017-present PyO3 Project and Contributors
 use proc_macro2::{Span, TokenStream};
 use quote::ToTokens;
-use syn::{spanned::Spanned, Ident};
+use syn::{punctuated::Punctuated, spanned::Spanned, Ident, Token};
 
 use crate::attributes::CrateAttribute;
 
@@ -57,70 +57,85 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> {
     None
 }
 
-/// A syntax tree which evaluates to a null-terminated docstring for Python.
+/// A syntax tree which evaluates to a nul-terminated docstring for Python.
 ///
-/// It's built as a `concat!` evaluation, so it's hard to do anything with this
+/// Typically the tokens will just be that string, but if the original docs included macro
+/// expressions then the tokens will be a concat!("...", "\n", "\0") expression of the strings and
+/// macro parts.
 /// contents such as parse the string contents.
 #[derive(Clone)]
 pub struct PythonDoc(TokenStream);
 
-/// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string
-/// e.g. concat!("...", "\n", "\0")
+/// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string.
 pub fn get_doc(
     attrs: &[syn::Attribute],
     text_signature: Option<(Cow<'_, Ident>, String)>,
 ) -> PythonDoc {
-    let mut tokens = TokenStream::new();
-    let comma = syn::token::Comma(Span::call_site());
-    let newline = syn::LitStr::new("\n", Span::call_site());
-
-    syn::Ident::new("concat", Span::call_site()).to_tokens(&mut tokens);
-    syn::token::Bang(Span::call_site()).to_tokens(&mut tokens);
-    syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| {
-        if let Some((python_name, text_signature)) = text_signature {
-            // create special doc string lines to set `__text_signature__`
-            let signature_lines = format!("{}{}\n--\n\n", python_name, text_signature);
-            signature_lines.to_tokens(tokens);
-            comma.to_tokens(tokens);
-        }
+    let mut parts = Punctuated::<TokenStream, Token![,]>::new();
+    let mut current_part = String::new();
+
+    if let Some((python_name, text_signature)) = text_signature {
+        // create special doc string lines to set `__text_signature__`
+        write!(
+            &mut current_part,
+            "{}{}\n--\n\n",
+            python_name, text_signature
+        )
+        .expect("error occurred while trying to format text_signature to string")
+    }
 
-        let mut first = true;
-
-        for attr in attrs.iter() {
-            if attr.path.is_ident("doc") {
-                if let Ok(DocArgs {
-                    _eq_token,
-                    token_stream,
-                }) = syn::parse2(attr.tokens.clone())
-                {
-                    if !first {
-                        newline.to_tokens(tokens);
-                        comma.to_tokens(tokens);
-                    } else {
-                        first = false;
-                    }
-                    if let Ok(syn::Lit::Str(lit_str)) = syn::parse2(token_stream.clone()) {
-                        // Strip single left space from literal strings, if needed.
-                        // e.g. `/// Hello world` expands to #[doc = " Hello world"]
-                        let doc_line = lit_str.value();
-                        doc_line
-                            .strip_prefix(' ')
-                            .map(|stripped| syn::LitStr::new(stripped, lit_str.span()))
-                            .unwrap_or(lit_str)
-                            .to_tokens(tokens);
-                    } else {
-                        // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)]
-                        token_stream.to_tokens(tokens)
-                    }
-                    comma.to_tokens(tokens);
+    let mut first = true;
+
+    for attr in attrs.iter() {
+        if attr.path.is_ident("doc") {
+            if let Ok(DocArgs {
+                _eq_token,
+                token_stream,
+            }) = syn::parse2(attr.tokens.clone())
+            {
+                if !first {
+                    current_part.push('\n');
+                } else {
+                    first = false;
+                }
+                if let Ok(syn::Lit::Str(lit_str)) = syn::parse2(token_stream.clone()) {
+                    // Strip single left space from literal strings, if needed.
+                    // e.g. `/// Hello world` expands to #[doc = " Hello world"]
+                    let doc_line = lit_str.value();
+                    current_part.push_str(doc_line.strip_prefix(' ').unwrap_or(&doc_line));
+                } else {
+                    // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)]
+                    // Reset the string buffer, write that part, and then push this macro part too.
+                    parts.push(current_part.to_token_stream());
+                    current_part.clear();
+                    parts.push(token_stream);
                 }
             }
         }
+    }
 
-        syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens);
-    });
+    if !parts.is_empty() {
+        // Doc contained macro pieces - return as `concat!` expression
+        if !current_part.is_empty() {
+            parts.push(current_part.to_token_stream());
+        }
 
-    PythonDoc(tokens)
+        let mut tokens = TokenStream::new();
+
+        syn::Ident::new("concat", Span::call_site()).to_tokens(&mut tokens);
+        syn::token::Bang(Span::call_site()).to_tokens(&mut tokens);
+        syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| {
+            parts.to_tokens(tokens);
+            syn::token::Comma(Span::call_site()).to_tokens(tokens);
+            syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens);
+        });
+
+        PythonDoc(tokens)
+    } else {
+        // Just a string doc - return directly with nul terminator
+        current_part.push('\0');
+        PythonDoc(current_part.to_token_stream())
+    }
 }
 
 impl quote::ToTokens for PythonDoc {

From 3c9ace03d8eeeb0b9cfb8efa842cb54140e4f494 Mon Sep 17 00:00:00 2001
From: Samuel Colvin <s@muelcolvin.com>
Date: Wed, 25 Jan 2023 11:05:14 +0000
Subject: [PATCH 14/31] add Ellipsis() and is_ellipsis() methods, fix #2906

---
 newsfragments/2911.added.md |  1 +
 pyo3-ffi/src/sliceobject.rs |  1 +
 src/instance.rs             | 23 +++++++++++++++++++++++
 src/marker.rs               | 18 ++++++++++++++++++
 src/types/any.rs            | 19 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 newsfragments/2911.added.md

diff --git a/newsfragments/2911.added.md b/newsfragments/2911.added.md
new file mode 100644
index 00000000000..4348412fee7
--- /dev/null
+++ b/newsfragments/2911.added.md
@@ -0,0 +1 @@
+Add `py.Ellipsis()` and `py_any.is_ellipsis()` methods.
diff --git a/pyo3-ffi/src/sliceobject.rs b/pyo3-ffi/src/sliceobject.rs
index 5f3138ef370..6b23f5ba278 100644
--- a/pyo3-ffi/src/sliceobject.rs
+++ b/pyo3-ffi/src/sliceobject.rs
@@ -2,6 +2,7 @@ use crate::object::*;
 use crate::pyport::Py_ssize_t;
 use std::os::raw::c_int;
 
+#[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {
     #[cfg_attr(PyPy, link_name = "_PyPy_EllipsisObject")]
     static mut _Py_EllipsisObject: PyObject;
diff --git a/src/instance.rs b/src/instance.rs
index b96c4cd5a42..fd937e5d1bc 100644
--- a/src/instance.rs
+++ b/src/instance.rs
@@ -519,6 +519,13 @@ impl<T> Py<T> {
         unsafe { ffi::Py_None() == self.as_ptr() }
     }
 
+    /// Returns whether the object is Ellipsis, e.g. `...`.
+    ///
+    /// This is equivalent to the Python expression `self is ...`.
+    pub fn is_ellipsis(&self) -> bool {
+        unsafe { ffi::Py_Ellipsis() == self.as_ptr() }
+    }
+
     /// Returns whether the object is considered to be true.
     ///
     /// This is equivalent to the Python expression `bool(self)`.
@@ -1147,6 +1154,22 @@ a = A()
         })
     }
 
+    #[test]
+    fn test_is_ellipsis() {
+        Python::with_gil(|py| {
+            let v = py
+                .eval("...", None, None)
+                .map_err(|e| e.print(py))
+                .unwrap()
+                .to_object(py);
+
+            assert!(v.is_ellipsis());
+
+            let not_ellipsis = 5.to_object(py);
+            assert!(!not_ellipsis.is_ellipsis());
+        });
+    }
+
     #[cfg(feature = "macros")]
     mod using_macros {
         use super::*;
diff --git a/src/marker.rs b/src/marker.rs
index 707402c906b..714af761020 100644
--- a/src/marker.rs
+++ b/src/marker.rs
@@ -619,6 +619,13 @@ impl<'py> Python<'py> {
         unsafe { PyObject::from_borrowed_ptr(self, ffi::Py_None()) }
     }
 
+    /// Gets the Python builtin value `Ellipsis`, or `...`.
+    #[allow(non_snake_case)] // the Python keyword starts with uppercase
+    #[inline]
+    pub fn Ellipsis(self) -> PyObject {
+        unsafe { PyObject::from_borrowed_ptr(self, ffi::Py_Ellipsis()) }
+    }
+
     /// Gets the Python builtin value `NotImplemented`.
     #[allow(non_snake_case)] // the Python keyword starts with uppercase
     #[inline]
@@ -1032,4 +1039,15 @@ mod tests {
         let state = unsafe { crate::ffi::PyGILState_Check() };
         assert_eq!(state, GIL_NOT_HELD);
     }
+
+    #[test]
+    fn test_ellipsis() {
+        Python::with_gil(|py| {
+            assert_eq!(py.Ellipsis().to_string(), "Ellipsis");
+
+            let v = py.eval("...", None, None).map_err(|e| e.print(py)).unwrap();
+
+            assert!(v.eq(py.Ellipsis()).unwrap());
+        });
+    }
 }
diff --git a/src/types/any.rs b/src/types/any.rs
index 5abe97854b3..6b3cd7a0be0 100644
--- a/src/types/any.rs
+++ b/src/types/any.rs
@@ -669,6 +669,13 @@ impl PyAny {
         unsafe { ffi::Py_None() == self.as_ptr() }
     }
 
+    /// Returns whether the object is Ellipsis, e.g. `...`.
+    ///
+    /// This is equivalent to the Python expression `self is ...`.
+    pub fn is_ellipsis(&self) -> bool {
+        unsafe { ffi::Py_Ellipsis() == self.as_ptr() }
+    }
+
     /// Returns true if the sequence or mapping has a length of 0.
     ///
     /// This is equivalent to the Python expression `len(self) == 0`.
@@ -1135,4 +1142,16 @@ class SimpleClass:
         let bools = [true, false];
         test_eq_methods_generic(&bools);
     }
+
+    #[test]
+    fn test_is_ellipsis() {
+        Python::with_gil(|py| {
+            let v = py.eval("...", None, None).map_err(|e| e.print(py)).unwrap();
+
+            assert!(v.is_ellipsis());
+
+            let not_ellipsis = 5.to_object(py).into_ref(py);
+            assert!(!not_ellipsis.is_ellipsis());
+        });
+    }
 }

From 1a4153f71831f8b7955efa93005ae2c960503e46 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 27 Jan 2023 08:20:03 +0000
Subject: [PATCH 15/31] remove unneeded into_iter calls

---
 benches/bench_set.rs                    | 2 +-
 pyo3-macros-backend/src/frompyobject.rs | 1 -
 pyo3-macros-backend/src/pymethod.rs     | 2 --
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/benches/bench_set.rs b/benches/bench_set.rs
index d59e3e57f4c..58abc956337 100644
--- a/benches/bench_set.rs
+++ b/benches/bench_set.rs
@@ -9,7 +9,7 @@ fn set_new(b: &mut Bencher<'_>) {
         const LEN: usize = 100_000;
         // Create Python objects up-front, so that the benchmark doesn't need to include
         // the cost of allocating LEN Python integers
-        let elements: Vec<PyObject> = (0..LEN).into_iter().map(|i| i.into_py(py)).collect();
+        let elements: Vec<PyObject> = (0..LEN).map(|i| i.into_py(py)).collect();
         b.iter(|| {
             let pool = unsafe { py.new_pool() };
             PySet::new(py, &elements).unwrap();
diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs
index ba365efe265..831aed253ff 100644
--- a/pyo3-macros-backend/src/frompyobject.rs
+++ b/pyo3-macros-backend/src/frompyobject.rs
@@ -278,7 +278,6 @@ impl<'a> Container<'a> {
         let self_ty = &self.path;
         let struct_name = &self.name();
         let field_idents: Vec<_> = (0..struct_fields.len())
-            .into_iter()
             .map(|i| format_ident!("arg{}", i))
             .collect();
         let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| {
diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs
index fa81c6d495d..3807a1fd8be 100644
--- a/pyo3-macros-backend/src/pymethod.rs
+++ b/pyo3-macros-backend/src/pymethod.rs
@@ -1109,7 +1109,6 @@ impl SlotDef {
         let py = syn::Ident::new("_py", Span::call_site());
         let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect();
         let arg_idents: &Vec<_> = &(0..arguments.len())
-            .into_iter()
             .map(|i| format_ident!("arg{}", i))
             .collect();
         let wrapper_ident = format_ident!("__pymethod_{}__", method_name);
@@ -1220,7 +1219,6 @@ impl SlotFragmentDef {
         let py = syn::Ident::new("_py", Span::call_site());
         let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect();
         let arg_idents: &Vec<_> = &(0..arguments.len())
-            .into_iter()
             .map(|i| format_ident!("arg{}", i))
             .collect();
         let body = generate_method_body(cls, spec, &py, arguments, *extract_error_mode, None)?;

From 2228f584a1ab4add68b111775e17b643a2361018 Mon Sep 17 00:00:00 2001
From: Alex Pyattaev <alex.pyattaev@gmail.com>
Date: Tue, 10 Jan 2023 23:42:07 +0200
Subject: [PATCH 16/31] added a plugin example that shows how to integrate a
 Python plugin into a Rust app while having option to test API without the
 main app

---
 examples/README.md                            |  1 +
 examples/plugin/Cargo.toml                    | 12 +++++
 examples/plugin/README.md                     | 48 +++++++++++++++++++
 examples/plugin/plugin_api/Cargo.toml         | 17 +++++++
 examples/plugin/plugin_api/noxfile.py         |  9 ++++
 examples/plugin/plugin_api/pyproject.toml     | 14 ++++++
 .../plugin/plugin_api/requirements-dev.txt    |  3 ++
 examples/plugin/plugin_api/src/lib.rs         | 32 +++++++++++++
 .../plugin/plugin_api/tests/test_Gadget.py    | 22 +++++++++
 .../plugin/plugin_api/tests/test_import.py    |  2 +
 .../python_plugin/gadget_init_plugin.py       | 12 +++++
 examples/plugin/python_plugin/rng.py          |  3 ++
 examples/plugin/src/main.rs                   | 44 +++++++++++++++++
 13 files changed, 219 insertions(+)
 create mode 100644 examples/plugin/Cargo.toml
 create mode 100644 examples/plugin/README.md
 create mode 100644 examples/plugin/plugin_api/Cargo.toml
 create mode 100644 examples/plugin/plugin_api/noxfile.py
 create mode 100644 examples/plugin/plugin_api/pyproject.toml
 create mode 100644 examples/plugin/plugin_api/requirements-dev.txt
 create mode 100644 examples/plugin/plugin_api/src/lib.rs
 create mode 100644 examples/plugin/plugin_api/tests/test_Gadget.py
 create mode 100644 examples/plugin/plugin_api/tests/test_import.py
 create mode 100644 examples/plugin/python_plugin/gadget_init_plugin.py
 create mode 100644 examples/plugin/python_plugin/rng.py
 create mode 100644 examples/plugin/src/main.rs

diff --git a/examples/README.md b/examples/README.md
index 7b1bd4d49c7..47ab5a9dc3a 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -10,6 +10,7 @@ Below is a brief description of each of these:
 | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. |
 | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. |
 | `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. |
+| `plugin` | Illustrates how to use Python as a scripting language within a Rust application |
 
 ## Creating new projects from these examples
 
diff --git a/examples/plugin/Cargo.toml b/examples/plugin/Cargo.toml
new file mode 100644
index 00000000000..08127b5003f
--- /dev/null
+++ b/examples/plugin/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "plugin_example"
+version = "0.1.0"
+edition = "2021"
+
+
+[dependencies]
+pyo3={path="../../", features=["macros"]}
+plugin_api={path="plugin_api"}
+
+
+[workspace]
diff --git a/examples/plugin/README.md b/examples/plugin/README.md
new file mode 100644
index 00000000000..17f2cf740bd
--- /dev/null
+++ b/examples/plugin/README.md
@@ -0,0 +1,48 @@
+# plugin
+
+An example of a Rust app that uses Python for a plugin. A Python extension module built using PyO3 and [`maturin`](https://github.com/PyO3/maturin) is used to provide
+interface types that can be used to exchange data between Rust and Python. This also deals with how to separately test and load python modules.
+
+# Building and Testing
+## Host application
+To run the app itself, you only need to run 
+
+```shell
+cargo run
+```
+It will build the app, as well as the plugin API, then run the app, load the plugin and show it working.
+
+## Plugin API testing
+
+The plugin API is in a separate crate `plugin_api`, so you can test it separately from the main app. 
+
+To build the API only package, first install `maturin`:
+
+```shell
+pip install maturin
+```
+
+When building the plugin, simply using `maturin develop` will fail to produce a viable extension module due to the features arrangement of PyO3. 
+Instead, one needs to enable the optional feature as follows:
+
+```shell
+cd plugin_api
+maturin build --features "extension-module"
+```
+
+Alternatively, install nox and run the tests inside an isolated environment:
+
+```shell
+nox
+```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/plugin
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)
diff --git a/examples/plugin/plugin_api/Cargo.toml b/examples/plugin/plugin_api/Cargo.toml
new file mode 100644
index 00000000000..870ad76a377
--- /dev/null
+++ b/examples/plugin/plugin_api/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "plugin_api"
+version = "0.1.0"
+description = "Plugin API example"
+edition = "2021"
+
+[lib]
+name = "plugin_api"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
+pyo3 = { path = "../../../" }
+
+[features]
+# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
+extension-module = ["pyo3/extension-module"]
diff --git a/examples/plugin/plugin_api/noxfile.py b/examples/plugin/plugin_api/noxfile.py
new file mode 100644
index 00000000000..3b53c0c3e36
--- /dev/null
+++ b/examples/plugin/plugin_api/noxfile.py
@@ -0,0 +1,9 @@
+import nox
+
+
+@nox.session
+def python(session):
+    session.install("-rrequirements-dev.txt")
+    session.install("maturin")
+    session.run_always("maturin", "develop", "--features", "extension-module")
+    session.run("pytest")
diff --git a/examples/plugin/plugin_api/pyproject.toml b/examples/plugin/plugin_api/pyproject.toml
new file mode 100644
index 00000000000..114687eddef
--- /dev/null
+++ b/examples/plugin/plugin_api/pyproject.toml
@@ -0,0 +1,14 @@
+[build-system]
+requires = ["maturin>=0.14,<0.15"]
+build-backend = "maturin"
+
+[project]
+name = "plugin_api"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+
+
diff --git a/examples/plugin/plugin_api/requirements-dev.txt b/examples/plugin/plugin_api/requirements-dev.txt
new file mode 100644
index 00000000000..20c7cdfbb1c
--- /dev/null
+++ b/examples/plugin/plugin_api/requirements-dev.txt
@@ -0,0 +1,3 @@
+pytest>=3.5.0
+pip>=21.3
+maturin>=0.14
diff --git a/examples/plugin/plugin_api/src/lib.rs b/examples/plugin/plugin_api/src/lib.rs
new file mode 100644
index 00000000000..59aae55699d
--- /dev/null
+++ b/examples/plugin/plugin_api/src/lib.rs
@@ -0,0 +1,32 @@
+use pyo3::prelude::*;
+
+///this is our Gadget that python plugin code can create, and rust app can then access natively.
+#[pyclass]
+pub struct Gadget {
+    #[pyo3(get, set)]
+    pub prop: usize,
+    //this field will only be accessible to rust code
+    pub rustonly: Vec<usize>,
+}
+
+#[pymethods]
+impl Gadget {
+    #[new]
+    fn new() -> Self {
+        Gadget {
+            prop: 777,
+            rustonly: Vec::new(),
+        }
+    }
+
+    fn push(&mut self, v: usize) {
+        self.rustonly.push(v);
+    }
+}
+
+/// A Python module for plugin interface types
+#[pymodule]
+pub fn plugin_api(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_class::<Gadget>()?;
+    Ok(())
+}
diff --git a/examples/plugin/plugin_api/tests/test_Gadget.py b/examples/plugin/plugin_api/tests/test_Gadget.py
new file mode 100644
index 00000000000..f1175f2789c
--- /dev/null
+++ b/examples/plugin/plugin_api/tests/test_Gadget.py
@@ -0,0 +1,22 @@
+import pytest
+
+
+@pytest.fixture
+def gadget():
+    import plugin_api as pa
+
+    g = pa.Gadget()
+    return g
+
+
+def test_creation(gadget):
+    pass
+
+
+def test_property(gadget):
+    gadget.prop = 42
+    assert gadget.prop == 42
+
+
+def test_push(gadget):
+    gadget.push(42)
diff --git a/examples/plugin/plugin_api/tests/test_import.py b/examples/plugin/plugin_api/tests/test_import.py
new file mode 100644
index 00000000000..ae1d6f67f6e
--- /dev/null
+++ b/examples/plugin/plugin_api/tests/test_import.py
@@ -0,0 +1,2 @@
+def test_import():
+    import plugin_api
diff --git a/examples/plugin/python_plugin/gadget_init_plugin.py b/examples/plugin/python_plugin/gadget_init_plugin.py
new file mode 100644
index 00000000000..2eeba6fa7b8
--- /dev/null
+++ b/examples/plugin/python_plugin/gadget_init_plugin.py
@@ -0,0 +1,12 @@
+import plugin_api
+import rng
+
+
+def start():
+    """create an instance of Gadget, configure it and return to Rust"""
+    g = plugin_api.Gadget()
+    g.push(1)
+    g.push(2)
+    g.push(3)
+    g.prop = rng.get_random_number()
+    return g
diff --git a/examples/plugin/python_plugin/rng.py b/examples/plugin/python_plugin/rng.py
new file mode 100644
index 00000000000..042e5e4b7d2
--- /dev/null
+++ b/examples/plugin/python_plugin/rng.py
@@ -0,0 +1,3 @@
+def get_random_number():
+    # verified by the roll of a fair die to be random
+    return 4
diff --git a/examples/plugin/src/main.rs b/examples/plugin/src/main.rs
new file mode 100644
index 00000000000..b50b54548e5
--- /dev/null
+++ b/examples/plugin/src/main.rs
@@ -0,0 +1,44 @@
+use plugin_api::plugin_api as pylib_module;
+use pyo3::prelude::*;
+use pyo3::types::PyList;
+use std::path::Path;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    //"export" our API module to the python runtime
+    pyo3::append_to_inittab!(pylib_module);
+    //spawn runtime
+    pyo3::prepare_freethreaded_python();
+    //import path for python
+    let path = Path::new("./python_plugin/");
+    //do useful work
+    Python::with_gil(|py| {
+        //add the current directory to import path of Python (do not use this in production!)
+        let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?;
+        syspath.insert(0, &path)?;
+        println!("Import path is: {:?}", syspath);
+
+        // Now we can load our python_plugin/gadget_init_plugin.py file.
+        // It can in turn import other stuff as it deems appropriate
+        let plugin = PyModule::import(py, "gadget_init_plugin")?;
+        // and call start function there, which will return a python reference to Gadget.
+        // Gadget here is a "pyclass" object reference
+        let gadget = plugin.getattr("start")?.call0()?;
+
+        //now we extract (i.e. mutably borrow) the rust struct from python object
+        {
+            //this scope will have mutable access to the gadget instance, which will be dropped on
+            //scope exit so Python can access it again.
+            let mut gadget_rs: PyRefMut<'_, plugin_api::Gadget> = gadget.extract()?;
+            // we can now modify it as if it was a native rust struct
+            gadget_rs.prop = 42;
+            //which includes access to rust-only fields that are not visible to python
+            println!("rust-only vec contains {:?}", gadget_rs.rustonly);
+            gadget_rs.rustonly.clear();
+        }
+
+        //any modifications we make to rust object are reflected on Python object as well
+        let res: usize = gadget.getattr("prop")?.extract()?;
+        println!("{res}");
+        Ok(())
+    })
+}

From d7601a43520df5ac90d5285f6214235a5a10d241 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 27 Jan 2023 08:31:44 +0000
Subject: [PATCH 17/31] add cargo-generate template for plugin example

---
 examples/plugin/.DS_Store                      | Bin 0 -> 6148 bytes
 examples/plugin/.template/Cargo.toml           |   9 +++++++++
 .../plugin/.template/plugin_api/Cargo.toml     |  17 +++++++++++++++++
 examples/plugin/.template/pre-script.rhai      |   4 ++++
 examples/plugin/cargo-generate.toml            |   5 +++++
 5 files changed, 35 insertions(+)
 create mode 100644 examples/plugin/.DS_Store
 create mode 100644 examples/plugin/.template/Cargo.toml
 create mode 100644 examples/plugin/.template/plugin_api/Cargo.toml
 create mode 100644 examples/plugin/.template/pre-script.rhai
 create mode 100644 examples/plugin/cargo-generate.toml

diff --git a/examples/plugin/.DS_Store b/examples/plugin/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..5336624e6244d43eb38f04aca66bd1af338e0754
GIT binary patch
literal 6148
zcmeHKF-`+P474FdP@0sK`vN3>u!@ow<O3iPr3nEcNPQLW;%Us-79zT&K}v(hl0Ca#
z&z^3I^V!UN@$qzTHaD{=oM@j6W8*n}WLK4OARKqR+Rh&B-8P5ysQPq*+#9<i$DV)l
z-}bwg{WfntO;uJ3NC7Dz1*Cu!_=5uMy|C#lQBev=0V(jK0KX3nPV9w4VthIfVgvwA
zkPgE-W(i<p0@w?OL}Xx|RA5rQMhs6n@~!H6;gFbg^BO+9p6oTDcsw2Vw<tI7iHcG{
z3LGnNo!dG4{|bI){y!#ZCk3Ryzf!;_%k^@BSE}ARdO7ym27iFF<_k{4Iw%Oyj)BpR
gv0*#DjiRh;T;seK4v9fWKIlOG3{V%D6u7hk-{ZCwS^xk5

literal 0
HcmV?d00001

diff --git a/examples/plugin/.template/Cargo.toml b/examples/plugin/.template/Cargo.toml
new file mode 100644
index 00000000000..e2017a20b8e
--- /dev/null
+++ b/examples/plugin/.template/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+pyo3 = "{{PYO3_VERSION}}"
+plugin_api = { path = "plugin_api" }
diff --git a/examples/plugin/.template/plugin_api/Cargo.toml b/examples/plugin/.template/plugin_api/Cargo.toml
new file mode 100644
index 00000000000..dfbd855aebc
--- /dev/null
+++ b/examples/plugin/.template/plugin_api/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "plugin_api"
+version = "0.1.0"
+description = "Plugin API example"
+edition = "2021"
+
+[lib]
+name = "plugin_api"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
+pyo3 = "{{PYO3_VERSION}}"
+
+[features]
+# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
+extension-module = ["pyo3/extension-module"]
diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai
new file mode 100644
index 00000000000..e126a37d9c0
--- /dev/null
+++ b/examples/plugin/.template/pre-script.rhai
@@ -0,0 +1,4 @@
+variable::set("PYO3_VERSION", "0.18.0");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml");
+file::delete(".template");
diff --git a/examples/plugin/cargo-generate.toml b/examples/plugin/cargo-generate.toml
new file mode 100644
index 00000000000..d750c4de7a3
--- /dev/null
+++ b/examples/plugin/cargo-generate.toml
@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

From cadcabaf043e1faa900e58f8810a9632e870c795 Mon Sep 17 00:00:00 2001
From: mejrs <59372212+mejrs@users.noreply.github.com>
Date: Sat, 28 Jan 2023 15:35:31 +0100
Subject: [PATCH 18/31] Feedback

---
 src/instance.rs  | 6 +++++-
 src/types/any.rs | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/instance.rs b/src/instance.rs
index 9c473c84b74..098571a0a82 100644
--- a/src/instance.rs
+++ b/src/instance.rs
@@ -1018,7 +1018,7 @@ impl PyObject {
     /// This is useful if you want to mutate a `PyObject` that
     /// might actually be a pyclass.
     ///
-    /// ```
+    /// ```rust
     /// # fn main() -> Result<(), pyo3::PyErr> {
     /// use pyo3::prelude::*;
     ///
@@ -1033,6 +1033,10 @@ impl PyObject {
     ///     let class_cell: &PyCell<Class> = class.downcast(py)?;
     ///
     ///     class_cell.borrow_mut().i += 1;
+    ///
+    ///     // Alternatively you can get a `PyRefMut` directly
+    ///     let class_ref: PyRefMut<'_, Class> = class.extract(py)?;
+    ///     assert_eq!(class_ref.i, 1);
     ///     Ok(())
     /// })
     /// # }
diff --git a/src/types/any.rs b/src/types/any.rs
index 9a57fc17600..6ef4feccb66 100644
--- a/src/types/any.rs
+++ b/src/types/any.rs
@@ -784,7 +784,7 @@ impl PyAny {
     /// This is useful if you want to mutate a `PyObject` that
     /// might actually be a pyclass.
     ///
-    /// ```
+    /// ```rust
     /// # fn main() -> Result<(), pyo3::PyErr> {
     /// use pyo3::prelude::*;
     ///
@@ -799,6 +799,10 @@ impl PyAny {
     ///     let class_cell: &PyCell<Class> = class.downcast()?;
     ///
     ///     class_cell.borrow_mut().i += 1;
+    ///
+    ///     // Alternatively you can get a `PyRefMut` directly
+    ///     let class_ref: PyRefMut<'_, Class> = class.extract()?;
+    ///     assert_eq!(class_ref.i, 1);
     ///     Ok(())
     /// })
     /// # }

From edaba66c445f862e757af403eeeb8278aabacd6e Mon Sep 17 00:00:00 2001
From: Adam Reichold <adam.reichold@t-online.de>
Date: Sat, 28 Jan 2023 18:24:37 +0100
Subject: [PATCH 19/31] Pin trybuild to an MSRV-compatible version

---
 noxfile.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/noxfile.py b/noxfile.py
index e054224bab6..bf6157337f3 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -436,6 +436,8 @@ def set_minimal_package_versions(session: nox.Session, venv_backend="none"):
         # 1.15.0 depends on hermit-abi 0.2.6 which has edition 2021 and breaks 1.48.0
         "num_cpus": "1.14.0",
         "parking_lot": "0.11.0",
+        # 1.0.77 needs basic-toml which has edition 2021
+        "trybuild": "1.0.76",
     }
 
     # run cargo update first to ensure that everything is at highest

From f8e2a2643933f3d0a5e6f3df057e9be4cc78a1ec Mon Sep 17 00:00:00 2001
From: messense <messense@icloud.com>
Date: Sun, 29 Jan 2023 09:50:32 +0800
Subject: [PATCH 20/31] Warn about unknown config keys in `PYO3_CONFIG_FILE`

---
 newsfragments/2926.changed.md   |  1 +
 pyo3-build-config/src/errors.rs |  3 +++
 pyo3-build-config/src/impl_.rs  | 24 +++++++++++++++++++++++-
 3 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/2926.changed.md

diff --git a/newsfragments/2926.changed.md b/newsfragments/2926.changed.md
new file mode 100644
index 00000000000..e0d4cfcc838
--- /dev/null
+++ b/newsfragments/2926.changed.md
@@ -0,0 +1 @@
+Warn about unknown config keys in `PYO3_CONFIG_FILE` instead of denying.
diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs
index 1cbf16ec476..6f57e0e6eed 100644
--- a/pyo3-build-config/src/errors.rs
+++ b/pyo3-build-config/src/errors.rs
@@ -19,6 +19,9 @@ macro_rules! warn {
     ($msg: literal) => {
         println!(concat!("cargo:warning=", $msg))
     };
+    ($fmt: expr, $($args: tt)+) => {
+        println!("cargo:warning={}", format_args!($fmt, $($args)+))
+    };
 }
 
 /// A simple error implementation which allows chaining of errors, inspired somewhat by anyhow.
diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs
index 2f0f5afda13..a46936cf7e7 100644
--- a/pyo3-build-config/src/impl_.rs
+++ b/pyo3-build-config/src/impl_.rs
@@ -433,7 +433,7 @@ print("mingw", get_platform().startswith("mingw"))
                 "extra_build_script_line" => {
                     extra_build_script_lines.push(value.to_string());
                 }
-                unknown => bail!("unknown config key `{}`", unknown),
+                unknown => warn!("unknown config key `{}`", unknown),
             }
         }
 
@@ -1865,6 +1865,28 @@ mod tests {
         )
     }
 
+    #[test]
+    fn test_config_file_unknown_keys() {
+        // ext_suffix is unknown to pyo3-build-config, but it shouldn't error
+        assert_eq!(
+            InterpreterConfig::from_reader("version=3.7\next_suffix=.python37.so".as_bytes())
+                .unwrap(),
+            InterpreterConfig {
+                version: PythonVersion { major: 3, minor: 7 },
+                implementation: PythonImplementation::CPython,
+                shared: true,
+                abi3: false,
+                lib_name: None,
+                lib_dir: None,
+                executable: None,
+                pointer_width: None,
+                build_flags: BuildFlags::default(),
+                suppress_build_script_link_lines: false,
+                extra_build_script_lines: vec![],
+            }
+        )
+    }
+
     #[test]
     fn build_flags_default() {
         assert_eq!(BuildFlags::default(), BuildFlags::new());

From f4953224d8ee71e4ee02570e8627306a607ba1af Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Wed, 25 Jan 2023 22:15:43 +0000
Subject: [PATCH 21/31] correct ffi definition of PyIter_Check

---
 newsfragments/2914.fixed.md |  1 +
 pyo3-ffi/src/abstract_.rs   | 11 ++++--
 src/types/iterator.rs       | 74 +++++++++++++++++++++++++++++++++++++
 3 files changed, 82 insertions(+), 4 deletions(-)
 create mode 100644 newsfragments/2914.fixed.md

diff --git a/newsfragments/2914.fixed.md b/newsfragments/2914.fixed.md
new file mode 100644
index 00000000000..9ea78ee84e8
--- /dev/null
+++ b/newsfragments/2914.fixed.md
@@ -0,0 +1 @@
+Fix downcast to `PyIterator` succeeding for Python classes which did not implement `__next__`.
diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs
index b8306ca9855..6f954bc1b47 100644
--- a/pyo3-ffi/src/abstract_.rs
+++ b/pyo3-ffi/src/abstract_.rs
@@ -91,9 +91,12 @@ extern "C" {
     pub fn PyObject_GetIter(arg1: *mut PyObject) -> *mut PyObject;
 }
 
-// Defined as this macro in Python limited API, but relies on
-// non-limited PyTypeObject. Don't expose this since it cannot be used.
-#[cfg(not(any(Py_LIMITED_API, PyPy)))]
+// Before 3.8 PyIter_Check was defined in CPython as a macro,
+// which uses Py_TYPE so cannot work on the limited ABI.
+//
+// From 3.10 onwards CPython removed the macro completely,
+// so PyO3 only uses this on 3.7 unlimited API.
+#[cfg(not(any(Py_3_8, Py_LIMITED_API, PyPy)))]
 #[inline]
 pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int {
     (match (*crate::Py_TYPE(o)).tp_iternext {
@@ -105,7 +108,7 @@ pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int {
 }
 
 extern "C" {
-    #[cfg(any(all(Py_3_8, Py_LIMITED_API), PyPy))]
+    #[cfg(any(Py_3_8, PyPy))]
     #[cfg_attr(PyPy, link_name = "PyPyIter_Check")]
     pub fn PyIter_Check(obj: *mut PyObject) -> c_int;
 
diff --git a/src/types/iterator.rs b/src/types/iterator.rs
index 5b51c0671a1..9d68ca4f666 100644
--- a/src/types/iterator.rs
+++ b/src/types/iterator.rs
@@ -248,4 +248,78 @@ def fibonacci(target):
             assert_eq!(iter_ref.get_refcnt(), 2);
         })
     }
+
+    #[test]
+    #[cfg(any(not(Py_LIMITED_API), Py_3_8))]
+    #[cfg(feature = "macros")]
+    fn python_class_not_iterator() {
+        use crate::PyErr;
+
+        #[crate::pyclass(crate = "crate")]
+        struct Downcaster {
+            failed: Option<PyErr>,
+        }
+
+        #[crate::pymethods(crate = "crate")]
+        impl Downcaster {
+            fn downcast_iterator(&mut self, obj: &PyAny) {
+                self.failed = Some(obj.downcast::<PyIterator>().unwrap_err().into());
+            }
+        }
+
+        // Regression test for 2913
+        Python::with_gil(|py| {
+            let downcaster = Py::new(py, Downcaster { failed: None }).unwrap();
+            crate::py_run!(
+                py,
+                downcaster,
+                r#"
+                    from collections.abc import Sequence
+
+                    class MySequence(Sequence):
+                        def __init__(self):
+                            self._data = [1, 2, 3]
+
+                        def __getitem__(self, index):
+                            return self._data[index]
+
+                        def __len__(self):
+                            return len(self._data)
+
+                    downcaster.downcast_iterator(MySequence())
+                "#
+            );
+
+            assert_eq!(
+                downcaster.borrow_mut(py).failed.take().unwrap().to_string(),
+                "TypeError: 'MySequence' object cannot be converted to 'Iterator'"
+            );
+        });
+    }
+
+    #[test]
+    #[cfg(any(not(Py_LIMITED_API), Py_3_8))]
+    #[cfg(feature = "macros")]
+    fn python_class_iterator() {
+        #[crate::pyfunction(crate = "crate")]
+        fn assert_iterator(obj: &PyAny) {
+            assert!(obj.downcast::<PyIterator>().is_ok())
+        }
+
+        // Regression test for 2913
+        Python::with_gil(|py| {
+            let assert_iterator = crate::wrap_pyfunction!(assert_iterator, py).unwrap();
+            crate::py_run!(
+                py,
+                assert_iterator,
+                r#"
+                    class MyIter:
+                        def __next__(self):
+                            raise StopIteration
+
+                    assert_iterator(MyIter())
+                "#
+            );
+        });
+    }
 }

From 066880e7d5ba9a414bc623854c49f09ed32b4f02 Mon Sep 17 00:00:00 2001
From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Wed, 18 Jan 2023 19:49:39 +0100
Subject: [PATCH 22/31] Added support for PyErr_WriteUnraisable

---
 newsfragments/2889.added.md |  1 +
 src/err/mod.rs              | 34 +++++++++++++++++++++++++++
 src/impl_/trampoline.rs     |  3 +--
 tests/test_exceptions.rs    | 47 +++++++++++++++++++++++++++++++++++++
 4 files changed, 83 insertions(+), 2 deletions(-)
 create mode 100644 newsfragments/2889.added.md

diff --git a/newsfragments/2889.added.md b/newsfragments/2889.added.md
new file mode 100644
index 00000000000..0d4c9265b1d
--- /dev/null
+++ b/newsfragments/2889.added.md
@@ -0,0 +1 @@
+Added `PyErr::write_unraisable()` to report an unraisable exception to Python.
diff --git a/src/err/mod.rs b/src/err/mod.rs
index 0f397153286..88b03986159 100644
--- a/src/err/mod.rs
+++ b/src/err/mod.rs
@@ -476,6 +476,40 @@ impl PyErr {
         unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) }
     }
 
+    /// Reports the error as unraisable.
+    ///
+    /// This calls `sys.unraisablehook()` using the current exception and obj argument.
+    ///
+    /// This method is useful to report errors in situations where there is no good mechanism
+    /// to report back to the Python land.  In Python this is used to indicate errors in
+    /// background threads or destructors which are protected.  In Rust code this is commonly
+    /// useful when you are calling into a Python callback which might fail, but there is no
+    /// obvious way to handle this error other than logging it.
+    ///
+    /// Calling this method has the benefit that the error goes back into a standardized callback
+    /// in Python which for instance allows unittests to ensure that no unraisable error
+    /// actually happend by hooking `sys.unraisablehook`.
+    ///
+    /// Example:
+    /// ```rust
+    /// # use pyo3::prelude::*;
+    /// # use pyo3::exceptions::PyRuntimeError;
+    /// # fn failing_function() -> PyResult<()> { Err(PyRuntimeError::new_err("foo")) }
+    /// # fn main() -> PyResult<()> {
+    /// Python::with_gil(|py| {
+    ///     match failing_function() {
+    ///         Err(pyerr) => pyerr.write_unraisable(py, None),
+    ///         Ok(..) => { /* do something here */ }
+    ///     }
+    ///     Ok(())
+    /// })
+    /// # }
+    #[inline]
+    pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) {
+        self.restore(py);
+        unsafe { ffi::PyErr_WriteUnraisable(obj.map_or(std::ptr::null_mut(), |x| x.as_ptr())) }
+    }
+
     /// Issues a warning message.
     ///
     /// May return an `Err(PyErr)` if warnings-as-errors is enabled.
diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs
index c75a68beacd..c7bea9abe2e 100644
--- a/src/impl_/trampoline.rs
+++ b/src/impl_/trampoline.rs
@@ -247,8 +247,7 @@ where
     if let Err(py_err) = panic::catch_unwind(move || body(py))
         .unwrap_or_else(|payload| Err(PanicException::from_panic_payload(payload)))
     {
-        py_err.restore(py);
-        ffi::PyErr_WriteUnraisable(ctx);
+        py_err.write_unraisable(py, py.from_borrowed_ptr_or_opt(ctx));
     }
     trap.disarm();
 }
diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs
index 98dab27bc70..cbbbd6368d5 100644
--- a/tests/test_exceptions.rs
+++ b/tests/test_exceptions.rs
@@ -96,3 +96,50 @@ fn test_exception_nosegfault() {
     assert!(io_err().is_err());
     assert!(parse_int().is_err());
 }
+
+#[test]
+#[cfg(Py_3_8)]
+fn test_write_unraisable() {
+    use pyo3::{exceptions::PyRuntimeError, ffi, AsPyPointer};
+
+    #[pyclass]
+    struct UnraisableCapture {
+        capture: Option<(PyErr, PyObject)>,
+    }
+
+    #[pymethods]
+    impl UnraisableCapture {
+        fn hook(&mut self, unraisable: &PyAny) {
+            let err = PyErr::from_value(unraisable.getattr("exc_value").unwrap());
+            let instance = unraisable.getattr("object").unwrap();
+            self.capture = Some((err, instance.into()));
+        }
+    }
+
+    Python::with_gil(|py| {
+        let sys = py.import("sys").unwrap();
+        let old_hook = sys.getattr("unraisablehook").unwrap();
+        let capture = Py::new(py, UnraisableCapture { capture: None }).unwrap();
+
+        sys.setattr("unraisablehook", capture.getattr(py, "hook").unwrap())
+            .unwrap();
+
+        assert!(capture.borrow(py).capture.is_none());
+
+        let err = PyRuntimeError::new_err("foo");
+        err.write_unraisable(py, None);
+
+        let (err, object) = capture.borrow_mut(py).capture.take().unwrap();
+        assert_eq!(err.to_string(), "RuntimeError: foo");
+        assert!(object.is_none(py));
+
+        let err = PyRuntimeError::new_err("bar");
+        err.write_unraisable(py, Some(py.NotImplemented().as_ref(py)));
+
+        let (err, object) = capture.borrow_mut(py).capture.take().unwrap();
+        assert_eq!(err.to_string(), "RuntimeError: bar");
+        assert!(object.as_ptr() == unsafe { ffi::Py_NotImplemented() });
+
+        sys.setattr("unraisablehook", old_hook).unwrap();
+    });
+}

From c09dfcd4e0197af843589c9d6fa9deff8b7a3b97 Mon Sep 17 00:00:00 2001
From: Samuel Colvin <s@muelcolvin.com>
Date: Wed, 25 Jan 2023 11:24:23 +0000
Subject: [PATCH 23/31] add PyDict.update() and PyDict.update_if_missing()

---
 newsfragments/2912.added.md |  1 +
 src/types/dict.rs           | 60 +++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 newsfragments/2912.added.md

diff --git a/newsfragments/2912.added.md b/newsfragments/2912.added.md
new file mode 100644
index 00000000000..0700a3af78a
--- /dev/null
+++ b/newsfragments/2912.added.md
@@ -0,0 +1 @@
+Add `PyDict.update()` and `PyDict.update_if_missing()` methods.
diff --git a/src/types/dict.rs b/src/types/dict.rs
index 35867ed1eb8..0cf7e6ca7b6 100644
--- a/src/types/dict.rs
+++ b/src/types/dict.rs
@@ -257,6 +257,28 @@ impl PyDict {
     pub fn as_mapping(&self) -> &PyMapping {
         unsafe { self.downcast_unchecked() }
     }
+
+    /// Update this dictionary with the key/value pairs from another.
+    ///
+    /// This is equivalent to the Python expression `self.update(other)`. If `other` is a `PyDict`, you may want
+    /// to use `self.update(other.as_mapping())`, note: `PyDict::as_mapping` is a zero-cost conversion.
+    pub fn update(&self, other: &PyMapping) -> PyResult<()> {
+        let py = self.py();
+        unsafe { err::error_on_minusone(py, ffi::PyDict_Update(self.as_ptr(), other.as_ptr())) }
+    }
+
+    /// Add key/value pairs from another dictionary to this one only when they do not exist in this.
+    ///
+    /// This is equivalent to the Python expression `self.update({k: v for k, v in other.items() if k not in self})`.
+    /// If `other` is a `PyDict`, you may want to use `self.update_if_missing(other.as_mapping())`,
+    /// note: `PyDict::as_mapping` is a zero-cost conversion.
+    ///
+    /// This method uses [`PyDict_Merge`](https://docs.python.org/3/c-api/dict.html#c.PyDict_Merge) internally,
+    /// so should have the same performance as `update`.
+    pub fn update_if_missing(&self, other: &PyMapping) -> PyResult<()> {
+        let py = self.py();
+        unsafe { err::error_on_minusone(py, ffi::PyDict_Merge(self.as_ptr(), other.as_ptr(), 0)) }
+    }
 }
 
 /// PyO3 implementation of an iterator for a Python `dict` object.
@@ -909,4 +931,42 @@ mod tests {
             assert!(items.is_instance(py.get_type::<PyDictItems>()).unwrap());
         })
     }
+
+    #[test]
+    fn dict_update() {
+        Python::with_gil(|py| {
+            let dict = [("a", 1), ("b", 2), ("c", 3)].into_py_dict(py);
+            let other = [("b", 4), ("c", 5), ("d", 6)].into_py_dict(py);
+            dict.update(other.as_mapping()).unwrap();
+            assert_eq!(dict.len(), 4);
+            assert_eq!(dict.get_item("a").unwrap().extract::<i32>().unwrap(), 1);
+            assert_eq!(dict.get_item("b").unwrap().extract::<i32>().unwrap(), 4);
+            assert_eq!(dict.get_item("c").unwrap().extract::<i32>().unwrap(), 5);
+            assert_eq!(dict.get_item("d").unwrap().extract::<i32>().unwrap(), 6);
+
+            assert_eq!(other.len(), 3);
+            assert_eq!(other.get_item("b").unwrap().extract::<i32>().unwrap(), 4);
+            assert_eq!(other.get_item("c").unwrap().extract::<i32>().unwrap(), 5);
+            assert_eq!(other.get_item("d").unwrap().extract::<i32>().unwrap(), 6);
+        })
+    }
+
+    #[test]
+    fn dict_update_if_missing() {
+        Python::with_gil(|py| {
+            let dict = [("a", 1), ("b", 2), ("c", 3)].into_py_dict(py);
+            let other = [("b", 4), ("c", 5), ("d", 6)].into_py_dict(py);
+            dict.update_if_missing(other.as_mapping()).unwrap();
+            assert_eq!(dict.len(), 4);
+            assert_eq!(dict.get_item("a").unwrap().extract::<i32>().unwrap(), 1);
+            assert_eq!(dict.get_item("b").unwrap().extract::<i32>().unwrap(), 2);
+            assert_eq!(dict.get_item("c").unwrap().extract::<i32>().unwrap(), 3);
+            assert_eq!(dict.get_item("d").unwrap().extract::<i32>().unwrap(), 6);
+
+            assert_eq!(other.len(), 3);
+            assert_eq!(other.get_item("b").unwrap().extract::<i32>().unwrap(), 4);
+            assert_eq!(other.get_item("c").unwrap().extract::<i32>().unwrap(), 5);
+            assert_eq!(other.get_item("d").unwrap().extract::<i32>().unwrap(), 6);
+        })
+    }
 }

From 26cf9b7163299db6069fd2fb9a08b6124caf8c35 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 3 Feb 2023 06:35:53 +0000
Subject: [PATCH 24/31] changelog: add missing entry for frozen

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 71123e40169..092edc23eb2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -124,6 +124,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h
 - Add `PyCode` and `PyFrame` high level objects. [#2408](https://github.com/PyO3/pyo3/pull/2408)
 - Add FFI definitions `Py_fstring_input`, `sendfunc`, and `_PyErr_StackItem`. [#2423](https://github.com/PyO3/pyo3/pull/2423)
 - Add `PyDateTime::new_with_fold`, `PyTime::new_with_fold`, `PyTime::get_fold`, and `PyDateTime::get_fold` for PyPy. [#2428](https://github.com/PyO3/pyo3/pull/2428)
+- Add `#[pyclass(frozen)]`. [#2448](https://github.com/PyO3/pyo3/pull/2448)
 - Accept `#[pyo3(name)]` on enum variants. [#2457](https://github.com/PyO3/pyo3/pull/2457)
 - Add `CompareOp::matches` to implement `__richcmp__` as the result of a Rust `std::cmp::Ordering` comparison. [#2460](https://github.com/PyO3/pyo3/pull/2460)
 - Add `PySuper` type. [#2486](https://github.com/PyO3/pyo3/pull/2486)

From 96efb0eda9cf4d9774037b779d4a7195685a2b42 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 3 Feb 2023 06:54:54 +0000
Subject: [PATCH 25/31] add better error message for Python in signature

---
 .../src/pyfunction/signature.rs               | 32 +++++++++++++------
 tests/ui/invalid_pyfunction_signatures.rs     |  5 +++
 tests/ui/invalid_pyfunction_signatures.stderr | 10 ++++--
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs
index 3a8527458ef..100da063a14 100644
--- a/pyo3-macros-backend/src/pyfunction/signature.rs
+++ b/pyo3-macros-backend/src/pyfunction/signature.rs
@@ -352,10 +352,21 @@ impl<'a> FunctionSignature<'a> {
         let mut parse_state = ParseState::Positional;
         let mut python_signature = PythonSignature::default();
 
-        let mut args_iter = arguments.iter_mut().filter(|arg| !arg.py); // Python<'_> arguments don't show on the Python side.
+        let mut args_iter = arguments.iter_mut();
+
+        let mut next_non_py_argument_checked = |name: &syn::Ident| {
+            for fn_arg in args_iter.by_ref() {
+                if fn_arg.py {
+                    // If the user incorrectly tried to include py: Python in the
+                    // signature, give a useful error as a hint.
+                    ensure_spanned!(
+                        name != fn_arg.name,
+                        name.span() => "arguments of type `Python` must not be part of the signature"
+                    );
+                    // Otherwise try next argument.
+                    continue;
+                }
 
-        let mut next_argument_checked = |name: &syn::Ident| match args_iter.next() {
-            Some(fn_arg) => {
                 ensure_spanned!(
                     name == fn_arg.name,
                     name.span() => format!(
@@ -364,17 +375,17 @@ impl<'a> FunctionSignature<'a> {
                         name.unraw(),
                     )
                 );
-                Ok(fn_arg)
+                return Ok(fn_arg);
             }
-            None => bail_spanned!(
+            bail_spanned!(
                 name.span() => "signature entry does not have a corresponding function argument"
-            ),
+            )
         };
 
         for item in &attribute.value.items {
             match item {
                 SignatureItem::Argument(arg) => {
-                    let fn_arg = next_argument_checked(&arg.ident)?;
+                    let fn_arg = next_non_py_argument_checked(&arg.ident)?;
                     parse_state.add_argument(
                         &mut python_signature,
                         arg.ident.unraw().to_string(),
@@ -389,12 +400,12 @@ impl<'a> FunctionSignature<'a> {
                     parse_state.finish_pos_args(&python_signature, sep.span())?
                 }
                 SignatureItem::Varargs(varargs) => {
-                    let fn_arg = next_argument_checked(&varargs.ident)?;
+                    let fn_arg = next_non_py_argument_checked(&varargs.ident)?;
                     fn_arg.is_varargs = true;
                     parse_state.add_varargs(&mut python_signature, &varargs)?;
                 }
                 SignatureItem::Kwargs(kwargs) => {
-                    let fn_arg = next_argument_checked(&kwargs.ident)?;
+                    let fn_arg = next_non_py_argument_checked(&kwargs.ident)?;
                     fn_arg.is_kwargs = true;
                     parse_state.add_kwargs(&mut python_signature, &kwargs)?;
                 }
@@ -404,7 +415,8 @@ impl<'a> FunctionSignature<'a> {
             };
         }
 
-        if let Some(arg) = args_iter.next() {
+        // Ensure no non-py arguments remain
+        if let Some(arg) = args_iter.find(|arg| !arg.py) {
             bail_spanned!(
                 attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name)
             );
diff --git a/tests/ui/invalid_pyfunction_signatures.rs b/tests/ui/invalid_pyfunction_signatures.rs
index 2bf849c0502..f5a9bee4e6c 100644
--- a/tests/ui/invalid_pyfunction_signatures.rs
+++ b/tests/ui/invalid_pyfunction_signatures.rs
@@ -44,6 +44,11 @@ fn function_with_kwargs_after_kwargs(kwargs_a: Option<&PyDict>, kwargs_b: Option
     let _ = kwargs_b;
 }
 
+#[pyfunction(signature = (py))]
+fn signature_contains_py(py: Python<'_>) {
+    let _ = py;
+}
+
 #[pyclass]
 struct MyClass;
 
diff --git a/tests/ui/invalid_pyfunction_signatures.stderr b/tests/ui/invalid_pyfunction_signatures.stderr
index 6720bb8753d..44031cceec7 100644
--- a/tests/ui/invalid_pyfunction_signatures.stderr
+++ b/tests/ui/invalid_pyfunction_signatures.stderr
@@ -46,8 +46,14 @@ error: `**kwargs_b` not allowed after `**kwargs_a`
 41 | #[pyo3(signature = (**kwargs_a, **kwargs_b))]
    |                                 ^
 
+error: arguments of type `Python` must not be part of the signature
+  --> tests/ui/invalid_pyfunction_signatures.rs:47:27
+   |
+47 | #[pyfunction(signature = (py))]
+   |                           ^^
+
 error: cannot define both function signature and legacy arguments
-  --> tests/ui/invalid_pyfunction_signatures.rs:53:12
+  --> tests/ui/invalid_pyfunction_signatures.rs:58:12
    |
-53 |     #[pyo3(signature = (x))]
+58 |     #[pyo3(signature = (x))]
    |            ^^^^^^^^^

From 5bab0e940925df4649fb9d4c0fbfbd80190ddecd Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 3 Feb 2023 07:53:38 +0000
Subject: [PATCH 26/31] use simplified PyIter_Check on CPython 3.7

---
 newsfragments/2914.changed.md |  1 +
 pyo3-ffi/src/abstract_.rs     | 18 ++++++++----------
 src/types/iterator.rs         |  7 +------
 3 files changed, 10 insertions(+), 16 deletions(-)
 create mode 100644 newsfragments/2914.changed.md

diff --git a/newsfragments/2914.changed.md b/newsfragments/2914.changed.md
new file mode 100644
index 00000000000..bea926c4976
--- /dev/null
+++ b/newsfragments/2914.changed.md
@@ -0,0 +1 @@
+FFI definition `PyIter_Check` on CPython 3.7 now does the equivalent for `hasattr(type(obj), "__next__")`, which works correctly on all platforms and adds support for `abi3`.
diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs
index 6f954bc1b47..24ad40057b1 100644
--- a/pyo3-ffi/src/abstract_.rs
+++ b/pyo3-ffi/src/abstract_.rs
@@ -92,19 +92,17 @@ extern "C" {
 }
 
 // Before 3.8 PyIter_Check was defined in CPython as a macro,
-// which uses Py_TYPE so cannot work on the limited ABI.
+// but the implementation of that in PyO3 did not work, see
+// https://github.com/PyO3/pyo3/pull/2914
 //
-// From 3.10 onwards CPython removed the macro completely,
-// so PyO3 only uses this on 3.7 unlimited API.
-#[cfg(not(any(Py_3_8, Py_LIMITED_API, PyPy)))]
+// This is a slow implementation which should function equivalently.
+#[cfg(not(any(Py_3_8, PyPy)))]
 #[inline]
 pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int {
-    (match (*crate::Py_TYPE(o)).tp_iternext {
-        Some(tp_iternext) => {
-            tp_iternext as *const std::os::raw::c_void != crate::_PyObject_NextNotImplemented as _
-        }
-        None => false,
-    }) as c_int
+    crate::PyObject_HasAttrString(
+        crate::Py_TYPE(o).cast(),
+        "__next__\0".as_ptr() as *const c_char,
+    )
 }
 
 extern "C" {
diff --git a/src/types/iterator.rs b/src/types/iterator.rs
index 9d68ca4f666..cfbbd31b6dc 100644
--- a/src/types/iterator.rs
+++ b/src/types/iterator.rs
@@ -3,7 +3,6 @@
 // based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
 
 use crate::{ffi, AsPyPointer, IntoPyPointer, Py, PyAny, PyErr, PyNativeType, PyResult, Python};
-#[cfg(any(not(Py_LIMITED_API), Py_3_8))]
 use crate::{PyDowncastError, PyTryFrom};
 
 /// A Python iterator object.
@@ -29,7 +28,6 @@ use crate::{PyDowncastError, PyTryFrom};
 #[repr(transparent)]
 pub struct PyIterator(PyAny);
 pyobject_native_type_named!(PyIterator);
-#[cfg(any(not(Py_LIMITED_API), Py_3_8))]
 pyobject_native_type_extract!(PyIterator);
 
 impl PyIterator {
@@ -64,7 +62,6 @@ impl<'p> Iterator for &'p PyIterator {
 }
 
 // PyIter_Check does not exist in the limited API until 3.8
-#[cfg(any(not(Py_LIMITED_API), Py_3_8))]
 impl<'v> PyTryFrom<'v> for PyIterator {
     fn try_from<V: Into<&'v PyAny>>(value: V) -> Result<&'v PyIterator, PyDowncastError<'v>> {
         let value = value.into();
@@ -218,7 +215,7 @@ def fibonacci(target):
     }
 
     #[test]
-    #[cfg(any(not(Py_LIMITED_API), Py_3_8))]
+
     fn iterator_try_from() {
         Python::with_gil(|py| {
             let obj: Py<PyAny> = vec![10, 20].to_object(py).as_ref(py).iter().unwrap().into();
@@ -250,7 +247,6 @@ def fibonacci(target):
     }
 
     #[test]
-    #[cfg(any(not(Py_LIMITED_API), Py_3_8))]
     #[cfg(feature = "macros")]
     fn python_class_not_iterator() {
         use crate::PyErr;
@@ -298,7 +294,6 @@ def fibonacci(target):
     }
 
     #[test]
-    #[cfg(any(not(Py_LIMITED_API), Py_3_8))]
     #[cfg(feature = "macros")]
     fn python_class_iterator() {
         #[crate::pyfunction(crate = "crate")]

From c709f30638c445f542925f9655a753f18322cab2 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 3 Feb 2023 07:26:18 +0000
Subject: [PATCH 27/31] guide: add documentation for trailing option arguments

---
 guide/src/function/signature.md | 65 ++++++++++++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md
index 651b4a3fc71..beefb02c23c 100644
--- a/guide/src/function/signature.md
+++ b/guide/src/function/signature.md
@@ -2,7 +2,7 @@
 
 The `#[pyfunction]` attribute also accepts parameters to control how the generated Python function accepts arguments. Just like in Python, arguments can be positional-only, keyword-only, or accept either. `*args` lists and `**kwargs` dicts can also be accepted. These parameters also work for `#[pymethods]` which will be introduced in the [Python Classes](../class.md) section of the guide.
 
-Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. There are two ways to modify this behaviour:
+Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. Most arguments are required by default, except for trailing `Option<_>` arguments, which are [implicitly given a default of `None`](#trailing-optional-arguments). There are two ways to modify this behaviour:
   - The `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax.
   - Extra arguments directly to `#[pyfunction]`. (See deprecated form)
 
@@ -106,6 +106,69 @@ num=-1
 > }
 > ```
 
+## Trailing optional arguments
+
+As a convenience, functions without a `#[pyo3(signature = (...))]` option will treat trailing `Option<T>` arguments as having a default of `None`. In the example below, PyO3 will create `increment` with a signature of `increment(x, amount=None)`.
+
+```rust
+use pyo3::prelude::*;
+
+/// Returns a copy of `x` increased by `amount`.
+///
+/// If `amount` is unspecified or `None`, equivalent to `x + 1`.
+#[pyfunction]
+fn increment(x: u64, amount: Option<u64>) -> u64 {
+    x + amount.unwrap_or(1)
+}
+#
+# fn main() -> PyResult<()> {
+#     Python::with_gil(|py| {
+#         let fun = pyo3::wrap_pyfunction!(increment, py)?;
+#
+#         let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
+#         let sig: String = inspect
+#             .call1((fun,))?
+#             .call_method0("__str__")?
+#             .extract()?;
+#
+#         #[cfg(Py_3_8)]  // on 3.7 the signature doesn't render b, upstream bug?
+#         assert_eq!(sig, "(x, amount=Ellipsis)");
+#
+#         Ok(())
+#     })
+# }
+```
+
+To make trailing `Option<T>` arguments required, but still accept `None`, add a `#[pyo3(signature = (...))]` annotation. For the example above, this would be `#[pyo3(signature = (x, amount))]`:
+
+```rust
+# use pyo3::prelude::*;
+#[pyfunction]
+#[pyo3(signature = (x, amount))]
+fn increment(x: u64, amount: Option<u64>) -> u64 {
+    x + amount.unwrap_or(1)
+}
+#
+# fn main() -> PyResult<()> {
+#     Python::with_gil(|py| {
+#         let fun = pyo3::wrap_pyfunction!(increment, py)?;
+#
+#         let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
+#         let sig: String = inspect
+#             .call1((fun,))?
+#             .call_method0("__str__")?
+#             .extract()?;
+#
+#         #[cfg(Py_3_8)]  // on 3.7 the signature doesn't render b, upstream bug?
+#         assert_eq!(sig, "(x, amount)");
+#
+#         Ok(())
+#     })
+# }
+```
+
+To help avoid confusion, PyO3 requires `#[pyo3(signature = (...))]` when an `Option<T>` argument is surrounded by arguments which aren't `Option<T>`.
+
 ## Deprecated form
 
 The `#[pyfunction]` macro can take the argument specification directly, but this method is deprecated in PyO3 0.18 because the `#[pyo3(signature)]` option offers a simpler syntax and better validation.

From 9306d56b275bd3c04d77bb0538438d31b17a4739 Mon Sep 17 00:00:00 2001
From: Julien Jerphanion <git@jjerphan.xyz>
Date: Sun, 29 Jan 2023 21:11:04 +0100
Subject: [PATCH 28/31] docs: Precise the abscense of `py: Python` for the
 #[pyo3(signature)]

---
 guide/src/function/signature.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md
index 651b4a3fc71..d8420ae4813 100644
--- a/guide/src/function/signature.md
+++ b/guide/src/function/signature.md
@@ -77,6 +77,22 @@ impl MyClass {
     }
 }
 ```
+
+Arguments of type `Python` must not be part of the signature:
+
+```rust
+# #![allow(dead_code)]
+# use pyo3::prelude::*;
+#[pyfunction]
+#[pyo3(signature = (lambda))]
+pub fn simple_python_bound_function(
+    py: Python<'_>,
+    lambda: PyObject,
+) -> PyResult<()> {
+    Ok(())
+}
+```
+
 N.B. the position of the `/` and `*` arguments (if included) control the system of handling positional and keyword arguments. In Python:
 ```python
 import mymodule

From bc1a33f9fede5ac49b65d0c9ff594a2f957abde5 Mon Sep 17 00:00:00 2001
From: Jeff Hodges <jeff@somethingsimilar.com>
Date: Sat, 4 Feb 2023 00:09:04 -0800
Subject: [PATCH 29/31] bump maturin version in getting_started.md

Latest maturin is 0.14.x
---
 guide/src/getting_started.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md
index 1628b627d7a..6f16cb72295 100644
--- a/guide/src/getting_started.md
+++ b/guide/src/getting_started.md
@@ -123,7 +123,7 @@ You should also create a `pyproject.toml` with the following contents:
 
 ```toml
 [build-system]
-requires = ["maturin>=0.13,<0.14"]
+requires = ["maturin>=0.14,<0.15"]
 build-backend = "maturin"
 
 [project]

From d67a8dc0a16acd3a52af81aed97a13280906c7c8 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Mon, 6 Feb 2023 20:33:16 +0000
Subject: [PATCH 30/31] link against pythonXY_d.dll for debug Python on Windows

---
 newsfragments/2937.fixed.md    |  1 +
 pyo3-build-config/src/impl_.rs | 51 ++++++++++++++++++++++++++++++----
 2 files changed, 46 insertions(+), 6 deletions(-)
 create mode 100644 newsfragments/2937.fixed.md

diff --git a/newsfragments/2937.fixed.md b/newsfragments/2937.fixed.md
new file mode 100644
index 00000000000..9956d7b9978
--- /dev/null
+++ b/newsfragments/2937.fixed.md
@@ -0,0 +1 @@
+Link against `pythonXY_d.dll` for debug Python builds on Windows.
diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs
index a46936cf7e7..159f999a6ec 100644
--- a/pyo3-build-config/src/impl_.rs
+++ b/pyo3-build-config/src/impl_.rs
@@ -230,6 +230,7 @@ print_if_set("base_prefix", base_prefix)
 print("executable", sys.executable)
 print("calcsize_pointer", struct.calcsize("P"))
 print("mingw", get_platform().startswith("mingw"))
+print("ext_suffix", get_config_var("EXT_SUFFIX"))
 "#;
         let output = run_python_script(interpreter.as_ref(), SCRIPT)?;
         let map: HashMap<String, String> = parse_script_output(&output);
@@ -261,6 +262,10 @@ print("mingw", get_platform().startswith("mingw"))
                 implementation,
                 abi3,
                 map["mingw"].as_str() == "True",
+                // This is the best heuristic currently available to detect debug build
+                // on Windows from sysconfig - e.g. ext_suffix may be
+                // `_d.cp312-win_amd64.pyd` for 3.12 debug build
+                map["ext_suffix"].starts_with("_d."),
             )
         } else {
             default_lib_name_unix(
@@ -1418,6 +1423,7 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf
             implementation,
             abi3,
             false,
+            false,
         ))
     } else {
         None
@@ -1493,6 +1499,7 @@ fn default_lib_name_for_target(
             implementation,
             abi3,
             false,
+            false,
         ))
     } else if is_linking_libpython_for_target(target) {
         Some(default_lib_name_unix(version, implementation, None))
@@ -1506,8 +1513,13 @@ fn default_lib_name_windows(
     implementation: PythonImplementation,
     abi3: bool,
     mingw: bool,
+    debug: bool,
 ) -> String {
-    if abi3 && !implementation.is_pypy() {
+    if debug {
+        // CPython bug: linking against python3_d.dll raises error
+        // https://github.com/python/cpython/issues/101614
+        format!("python{}{}_d", version.major, version.minor)
+    } else if abi3 && !implementation.is_pypy() {
         WINDOWS_ABI3_LIB_NAME.to_owned()
     } else if mingw {
         // https://packages.msys2.org/base/mingw-w64-python
@@ -2244,7 +2256,8 @@ mod tests {
                 PythonVersion { major: 3, minor: 7 },
                 CPython,
                 false,
-                false
+                false,
+                false,
             ),
             "python37",
         );
@@ -2253,7 +2266,8 @@ mod tests {
                 PythonVersion { major: 3, minor: 7 },
                 CPython,
                 true,
-                false
+                false,
+                false,
             ),
             "python3",
         );
@@ -2262,7 +2276,8 @@ mod tests {
                 PythonVersion { major: 3, minor: 7 },
                 CPython,
                 false,
-                true
+                true,
+                false,
             ),
             "python3.7",
         );
@@ -2271,7 +2286,8 @@ mod tests {
                 PythonVersion { major: 3, minor: 7 },
                 CPython,
                 true,
-                true
+                true,
+                false,
             ),
             "python3",
         );
@@ -2280,10 +2296,33 @@ mod tests {
                 PythonVersion { major: 3, minor: 7 },
                 PyPy,
                 true,
-                false
+                false,
+                false,
             ),
             "python37",
         );
+        assert_eq!(
+            super::default_lib_name_windows(
+                PythonVersion { major: 3, minor: 7 },
+                CPython,
+                false,
+                false,
+                true,
+            ),
+            "python37_d",
+        );
+        // abi3 debug builds on windows use version-specific lib
+        // to workaround https://github.com/python/cpython/issues/101614
+        assert_eq!(
+            super::default_lib_name_windows(
+                PythonVersion { major: 3, minor: 7 },
+                CPython,
+                true,
+                false,
+                true,
+            ),
+            "python37_d",
+        );
     }
 
     #[test]

From 37d377ea6cf5ba3999998f75046964023bd13845 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Fri, 3 Feb 2023 07:11:01 +0000
Subject: [PATCH 31/31] release: 0.18.1

---
 CHANGELOG.md                                  | 25 ++++++++++++++++++-
 Cargo.toml                                    |  8 +++---
 README.md                                     |  4 +--
 examples/Cargo.toml                           |  2 +-
 examples/decorator/.template/pre-script.rhai  |  2 +-
 .../maturin-starter/.template/pre-script.rhai |  2 +-
 examples/plugin/.template/pre-script.rhai     |  2 +-
 .../.template/pre-script.rhai                 |  2 +-
 examples/word-count/.template/pre-script.rhai |  2 +-
 newsfragments/2886.fixed.md                   |  1 -
 newsfragments/2889.added.md                   |  1 -
 newsfragments/2911.added.md                   |  1 -
 newsfragments/2912.added.md                   |  1 -
 newsfragments/2914.changed.md                 |  1 -
 newsfragments/2914.fixed.md                   |  1 -
 newsfragments/2921.fixed.md                   |  1 -
 newsfragments/2923.fixed.md                   |  1 -
 newsfragments/2926.changed.md                 |  1 -
 newsfragments/2937.fixed.md                   |  1 -
 pyo3-build-config/Cargo.toml                  |  2 +-
 pyo3-ffi/Cargo.toml                           |  4 +--
 pyo3-macros-backend/Cargo.toml                |  2 +-
 pyo3-macros/Cargo.toml                        |  4 +--
 pyproject.toml                                |  2 +-
 24 files changed, 43 insertions(+), 30 deletions(-)
 delete mode 100644 newsfragments/2886.fixed.md
 delete mode 100644 newsfragments/2889.added.md
 delete mode 100644 newsfragments/2911.added.md
 delete mode 100644 newsfragments/2912.added.md
 delete mode 100644 newsfragments/2914.changed.md
 delete mode 100644 newsfragments/2914.fixed.md
 delete mode 100644 newsfragments/2921.fixed.md
 delete mode 100644 newsfragments/2923.fixed.md
 delete mode 100644 newsfragments/2926.changed.md
 delete mode 100644 newsfragments/2937.fixed.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 092edc23eb2..46b9b0286f6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,28 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h
 
 <!-- towncrier release notes start -->
 
+## [0.18.1] - 2023-02-07
+
+### Added
+
+- Add `PyErr::write_unraisable()`. [#2889](https://github.com/PyO3/pyo3/pull/2889)
+- Add `Python::Ellipsis()` and `PyAny::is_ellipsis()` methods. [#2911](https://github.com/PyO3/pyo3/pull/2911)
+- Add `PyDict::update()` and `PyDict::update_if_missing()` methods. [#2912](https://github.com/PyO3/pyo3/pull/2912)
+
+### Changed
+
+- FFI definition `PyIter_Check` on CPython 3.7 is now implemented as `hasattr(type(obj), "__next__")`, which works correctly on all platforms and adds support for `abi3`. [#2914](https://github.com/PyO3/pyo3/pull/2914)
+- Warn about unknown config keys in `PYO3_CONFIG_FILE` instead of denying. [#2926](https://github.com/PyO3/pyo3/pull/2926)
+
+### Fixed
+
+- Send errors returned by `__releasebuffer__` to `sys.unraisablehook` rather than causing `SystemError`. [#2886](https://github.com/PyO3/pyo3/pull/2886)
+- Fix downcast to `PyIterator` succeeding for Python classes which did not implement `__next__`. [#2914](https://github.com/PyO3/pyo3/pull/2914)
+- Fix segfault in `__traverse__` when visiting `None` fields of `Option<T: AsPyPointer>`. [#2921](https://github.com/PyO3/pyo3/pull/2921)
+- Fix `#[pymethods(crate = "...")]` option being ignored. [#2923](https://github.com/PyO3/pyo3/pull/2923)
+- Link against `pythonXY_d.dll` for debug Python builds on Windows. [#2937](https://github.com/PyO3/pyo3/pull/2937)
+
+
 ## [0.18.0] - 2023-01-17
 
 ### Packaging
@@ -1370,7 +1392,8 @@ Yanked
 
 - Initial release
 
-[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.18.0...HEAD
+[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.18.1...HEAD
+[0.18.1]: https://github.com/pyo3/pyo3/compare/v0.18.0...v0.18.1
 [0.18.0]: https://github.com/pyo3/pyo3/compare/v0.17.3...v0.18.0
 [0.17.3]: https://github.com/pyo3/pyo3/compare/v0.17.2...v0.17.3
 [0.17.2]: https://github.com/pyo3/pyo3/compare/v0.17.1...v0.17.2
diff --git a/Cargo.toml b/Cargo.toml
index b6946f6fe8c..2431c4ac58a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3"
-version = "0.18.0"
+version = "0.18.1"
 description = "Bindings to Python interpreter"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 readme = "README.md"
@@ -20,10 +20,10 @@ parking_lot = ">= 0.11, < 0.13"
 memoffset = "0.8"
 
 # ffi bindings to the python interpreter, split into a separate crate so they can be used independently
-pyo3-ffi = { path = "pyo3-ffi", version = "=0.18.0" }
+pyo3-ffi = { path = "pyo3-ffi", version = "=0.18.1" }
 
 # support crates for macros feature
-pyo3-macros = { path = "pyo3-macros", version = "=0.18.0", optional = true }
+pyo3-macros = { path = "pyo3-macros", version = "=0.18.1", optional = true }
 indoc = { version = "1.0.3", optional = true }
 unindent = { version = "0.1.4", optional = true }
 
@@ -56,7 +56,7 @@ rayon = "1.0.2"
 widestring = "0.5.1"
 
 [build-dependencies]
-pyo3-build-config = { path = "pyo3-build-config", version = "0.18.0", features = ["resolve-config"] }
+pyo3-build-config = { path = "pyo3-build-config", version = "0.18.1", features = ["resolve-config"] }
 
 [features]
 default = ["macros"]
diff --git a/README.md b/README.md
index 63c125e1426..1c94e5d2511 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ name = "string_sum"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.18.0", features = ["extension-module"] }
+pyo3 = { version = "0.18.1", features = ["extension-module"] }
 ```
 
 **`src/lib.rs`**
@@ -137,7 +137,7 @@ Start a new project with `cargo new` and add  `pyo3` to the `Cargo.toml` like th
 
 ```toml
 [dependencies.pyo3]
-version = "0.18.0"
+version = "0.18.1"
 features = ["auto-initialize"]
 ```
 
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 52a8069ff2d..b178632dbb3 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -5,7 +5,7 @@ publish = false
 edition = "2018"
 
 [dev-dependencies]
-pyo3 = { version = "0.18.0", path = "..", features = ["auto-initialize", "extension-module"] }
+pyo3 = { version = "0.18.1", path = "..", features = ["auto-initialize", "extension-module"] }
 
 [[example]]
 name = "decorator"
diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai
index c9759acc70e..f2ff445e23a 100644
--- a/examples/decorator/.template/pre-script.rhai
+++ b/examples/decorator/.template/pre-script.rhai
@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.18.0");
+variable::set("PYO3_VERSION", "0.18.1");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/pyproject.toml", "pyproject.toml");
 file::rename(".template/tox.ini", "tox.ini");
diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai
index c9759acc70e..f2ff445e23a 100644
--- a/examples/maturin-starter/.template/pre-script.rhai
+++ b/examples/maturin-starter/.template/pre-script.rhai
@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.18.0");
+variable::set("PYO3_VERSION", "0.18.1");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/pyproject.toml", "pyproject.toml");
 file::rename(".template/tox.ini", "tox.ini");
diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai
index e126a37d9c0..e32afb1001d 100644
--- a/examples/plugin/.template/pre-script.rhai
+++ b/examples/plugin/.template/pre-script.rhai
@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.18.0");
+variable::set("PYO3_VERSION", "0.18.1");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml");
 file::delete(".template");
diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai
index 69574a02fdb..17799f1cb08 100644
--- a/examples/setuptools-rust-starter/.template/pre-script.rhai
+++ b/examples/setuptools-rust-starter/.template/pre-script.rhai
@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.18.0");
+variable::set("PYO3_VERSION", "0.18.1");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/setup.cfg", "setup.cfg");
 file::rename(".template/tox.ini", "tox.ini");
diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai
index 88f41205d08..5aeee32bdb7 100644
--- a/examples/word-count/.template/pre-script.rhai
+++ b/examples/word-count/.template/pre-script.rhai
@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.18.0");
+variable::set("PYO3_VERSION", "0.18.1");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/tox.ini", "tox.ini");
 file::delete(".template");
diff --git a/newsfragments/2886.fixed.md b/newsfragments/2886.fixed.md
deleted file mode 100644
index cfe192d4ef0..00000000000
--- a/newsfragments/2886.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Send errors returned by `__releasebuffer__` to `sys.unraisablehook` rather than causing `SystemError`.
diff --git a/newsfragments/2889.added.md b/newsfragments/2889.added.md
deleted file mode 100644
index 0d4c9265b1d..00000000000
--- a/newsfragments/2889.added.md
+++ /dev/null
@@ -1 +0,0 @@
-Added `PyErr::write_unraisable()` to report an unraisable exception to Python.
diff --git a/newsfragments/2911.added.md b/newsfragments/2911.added.md
deleted file mode 100644
index 4348412fee7..00000000000
--- a/newsfragments/2911.added.md
+++ /dev/null
@@ -1 +0,0 @@
-Add `py.Ellipsis()` and `py_any.is_ellipsis()` methods.
diff --git a/newsfragments/2912.added.md b/newsfragments/2912.added.md
deleted file mode 100644
index 0700a3af78a..00000000000
--- a/newsfragments/2912.added.md
+++ /dev/null
@@ -1 +0,0 @@
-Add `PyDict.update()` and `PyDict.update_if_missing()` methods.
diff --git a/newsfragments/2914.changed.md b/newsfragments/2914.changed.md
deleted file mode 100644
index bea926c4976..00000000000
--- a/newsfragments/2914.changed.md
+++ /dev/null
@@ -1 +0,0 @@
-FFI definition `PyIter_Check` on CPython 3.7 now does the equivalent for `hasattr(type(obj), "__next__")`, which works correctly on all platforms and adds support for `abi3`.
diff --git a/newsfragments/2914.fixed.md b/newsfragments/2914.fixed.md
deleted file mode 100644
index 9ea78ee84e8..00000000000
--- a/newsfragments/2914.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix downcast to `PyIterator` succeeding for Python classes which did not implement `__next__`.
diff --git a/newsfragments/2921.fixed.md b/newsfragments/2921.fixed.md
deleted file mode 100644
index 8cbffabb3ce..00000000000
--- a/newsfragments/2921.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Traversal visit calls to `Option<T: AsPyPointer>` no longer segfaults when `None`.
diff --git a/newsfragments/2923.fixed.md b/newsfragments/2923.fixed.md
deleted file mode 100644
index d61f5b01e65..00000000000
--- a/newsfragments/2923.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix `#[pymethods(crate = "...")]` option being ignored.
diff --git a/newsfragments/2926.changed.md b/newsfragments/2926.changed.md
deleted file mode 100644
index e0d4cfcc838..00000000000
--- a/newsfragments/2926.changed.md
+++ /dev/null
@@ -1 +0,0 @@
-Warn about unknown config keys in `PYO3_CONFIG_FILE` instead of denying.
diff --git a/newsfragments/2937.fixed.md b/newsfragments/2937.fixed.md
deleted file mode 100644
index 9956d7b9978..00000000000
--- a/newsfragments/2937.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Link against `pythonXY_d.dll` for debug Python builds on Windows.
diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml
index 5aa756941a5..a1d48bb74c1 100644
--- a/pyo3-build-config/Cargo.toml
+++ b/pyo3-build-config/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-build-config"
-version = "0.18.0"
+version = "0.18.1"
 description = "Build configuration for the PyO3 ecosystem"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 keywords = ["pyo3", "python", "cpython", "ffi"]
diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml
index 80bf3eb8100..aba3bf22704 100644
--- a/pyo3-ffi/Cargo.toml
+++ b/pyo3-ffi/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-ffi"
-version = "0.18.0"
+version = "0.18.1"
 description = "Python-API bindings for the PyO3 ecosystem"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 keywords = ["pyo3", "python", "cpython", "ffi"]
@@ -38,4 +38,4 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"]
 
 
 [build-dependencies]
-pyo3-build-config = { path = "../pyo3-build-config", version = "0.18.0", features = ["resolve-config"] }
+pyo3-build-config = { path = "../pyo3-build-config", version = "0.18.1", features = ["resolve-config"] }
diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml
index 3ff76d84273..d3283aa2102 100644
--- a/pyo3-macros-backend/Cargo.toml
+++ b/pyo3-macros-backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-macros-backend"
-version = "0.18.0"
+version = "0.18.1"
 description = "Code generation for PyO3 package"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 keywords = ["pyo3", "python", "cpython", "ffi"]
diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml
index 661de9ca072..496e73d7a5f 100644
--- a/pyo3-macros/Cargo.toml
+++ b/pyo3-macros/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-macros"
-version = "0.18.0"
+version = "0.18.1"
 description = "Proc macros for PyO3 package"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 keywords = ["pyo3", "python", "cpython", "ffi"]
@@ -22,4 +22,4 @@ abi3 = ["pyo3-macros-backend/abi3"]
 proc-macro2 = { version = "1", default-features = false }
 quote = "1"
 syn = { version = "1.0.56", features = ["full", "extra-traits"] }
-pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.18.0" }
+pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.18.1" }
diff --git a/pyproject.toml b/pyproject.toml
index e62881d25e1..9fabaf77dbb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,7 +20,7 @@ exclude = '''
 
 [tool.towncrier]
 filename = "CHANGELOG.md"
-version = "0.18.0"
+version = "0.18.1"
 start_string = "<!-- towncrier release notes start -->\n"
 template = ".towncrier.template.md"
 title_format = "## [{version}] - {project_date}"