Improve Python 3.11 support (#3694)

* Test out Python 3.11 migration

* Clean up a bit

* Remove todo

* Test workaround

* Fix potential bug uncovered in 3.11

* Try to fix it more

* last ditch fix

* Revert. Tp-traverse isn't the problem

* Test workaround

* Try this hack

* Revert MRO changes

* Use f_back properly

* Qualify auto

* Update include/pybind11/pybind11.h

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Simplify code slightly

* Ensure co_varnames decref if dict_getitem throws

* Eager decref f_code

Co-authored-by: Henry Schreiner <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

include/pybind11/detail/type_caster_base.h

@@ -502,9 +502,10 @@ PYBIND11_NOINLINE std::string error_string() {
         }
 
         PyFrameObject *frame = trace->tb_frame;
+        Py_XINCREF(frame);
         errorString += "\n\nAt:\n";
         while (frame) {
-#    if PY_VERSION_HEX >= 0x03090000
+#    if PY_VERSION_HEX >= 0x030900B1
             PyCodeObject *f_code = PyFrame_GetCode(frame);
 #    else
             PyCodeObject *f_code = frame->f_code;
@@ -514,8 +515,15 @@ PYBIND11_NOINLINE std::string error_string() {
             errorString += "  " + handle(f_code->co_filename).cast<std::string>() + "("
                            + std::to_string(lineno)
                            + "): " + handle(f_code->co_name).cast<std::string>() + "\n";
-            frame = frame->f_back;
             Py_DECREF(f_code);
+#    if PY_VERSION_HEX >= 0x030900B1
+            auto *b_frame = PyFrame_GetBack(frame);
+#    else
+            auto *b_frame = frame->f_back;
+            Py_XINCREF(b_frame);
+#    endif
+            Py_DECREF(frame);
+            frame = b_frame;
         }
     }
 #endif

include/pybind11/pybind11.h

@@ -2636,19 +2636,19 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
 
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
-#if !defined(PYPY_VERSION) && PY_VERSION_HEX < 0x030B0000
-    // TODO: Remove PyPy workaround for Python 3.11.
-    // Current API fails on 3.11 since co_varnames can be null.
+#if !defined(PYPY_VERSION)
 #    if PY_VERSION_HEX >= 0x03090000
     PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
     if (frame != nullptr) {
         PyCodeObject *f_code = PyFrame_GetCode(frame);
         // f_code is guaranteed to not be NULL
         if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
             PyObject *locals = PyEval_GetLocals();
-            if (locals != nullptr && f_code->co_varnames != nullptr) {
-                PyObject *self_caller
-                    = dict_getitem(locals, PyTuple_GET_ITEM(f_code->co_varnames, 0));
+            if (locals != nullptr) {
+                PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames");
+                PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0);
+                Py_DECREF(co_varnames);
+                PyObject *self_caller = dict_getitem(locals, self_arg);
                 if (self_caller == self.ptr()) {
                     Py_DECREF(f_code);
                     Py_DECREF(frame);