From 0290852156fdbbabf1152ac7521a08944a8aa02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Thu, 21 Sep 2023 16:21:14 +0200 Subject: [PATCH 01/10] MAINT: Include numpy._core imports --- include/pybind11/numpy.h | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 02f65d740d..18485ba1e6 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -263,7 +263,12 @@ struct npy_api { }; static npy_api lookup() { - module_ m = module_::import("numpy.core.multiarray"); + module_ m; + try { + m = module_::import("numpy._core.multiarray"); + } catch (error_already_set&) { + m = module_::import("numpy.core.multiarray"); + } auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); npy_api api; @@ -626,8 +631,13 @@ class dtype : public object { private: static object _dtype_from_pep3118() { - static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118") + module_ m; + try { + m = module_::import("numpy._core._internal"); + } catch (error_already_set&) { + m = module_::import("numpy.core._internal"); + } + static PyObject *obj = m.attr("_dtype_from_pep3118") .cast() .release() .ptr(); From e8d98f8547c02465397aa8eb59551b16be6df69e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:36:18 +0000 Subject: [PATCH 02/10] style: pre-commit fixes --- include/pybind11/numpy.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 18485ba1e6..a03b248784 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -266,7 +266,7 @@ struct npy_api { module_ m; try { m = module_::import("numpy._core.multiarray"); - } catch (error_already_set&) { + } catch (error_already_set &) { m = module_::import("numpy.core.multiarray"); } auto c = m.attr("_ARRAY_API"); @@ -634,13 +634,10 @@ class dtype : public object { module_ m; try { m = module_::import("numpy._core._internal"); - } catch (error_already_set&) { + } catch (error_already_set &) { m = module_::import("numpy.core._internal"); } - static PyObject *obj = m.attr("_dtype_from_pep3118") - .cast() - .release() - .ptr(); + static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); return reinterpret_borrow(obj); } From ee40dab3d2eb4e55aed9c647fb4961bffd182fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Tue, 26 Sep 2023 12:28:39 +0200 Subject: [PATCH 03/10] Apply review comments --- include/pybind11/numpy.h | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a03b248784..3c70f37a21 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,6 +120,26 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } +module_ import_numpy_core_submodule(const char * submodule_name) { + try { + return module_::import( + (std::string("numpy._core.") + submodule_name).c_str() + ); + } catch (error_already_set &ex) { + if (!ex.matches(PyExc_ImportError)) throw; + try { + return module_::import( + (std::string("numpy.core.") + submodule_name).c_str() + ); + } catch(error_already_set &ex) { + if (!ex.matches(PyExc_ImportError)) throw; + throw import_error( + std::string("pybind11 couldn't import ") + submodule_name + " from numpy." + ); + } + } +} + template struct same_size { template @@ -263,12 +283,7 @@ struct npy_api { }; static npy_api lookup() { - module_ m; - try { - m = module_::import("numpy._core.multiarray"); - } catch (error_already_set &) { - m = module_::import("numpy.core.multiarray"); - } + module_ m = import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); npy_api api; @@ -631,13 +646,11 @@ class dtype : public object { private: static object _dtype_from_pep3118() { - module_ m; - try { - m = module_::import("numpy._core._internal"); - } catch (error_already_set &) { - m = module_::import("numpy.core._internal"); - } - static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); + module_ m = import_numpy_core_submodule("_internal"); + static PyObject *obj = m.attr("_dtype_from_pep3118") + .cast() + .release() + .ptr(); return reinterpret_borrow(obj); } From 340afabe800235c6eabe2de4e570fd8a2d89bc2e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:31:47 +0000 Subject: [PATCH 04/10] style: pre-commit fixes --- include/pybind11/numpy.h | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3c70f37a21..81c39ec840 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,22 +120,19 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } -module_ import_numpy_core_submodule(const char * submodule_name) { +module_ import_numpy_core_submodule(const char *submodule_name) { try { - return module_::import( - (std::string("numpy._core.") + submodule_name).c_str() - ); + return module_::import((std::string("numpy._core.") + submodule_name).c_str()); } catch (error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) throw; + if (!ex.matches(PyExc_ImportError)) + throw; try { - return module_::import( - (std::string("numpy.core.") + submodule_name).c_str() - ); - } catch(error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) throw; - throw import_error( - std::string("pybind11 couldn't import ") + submodule_name + " from numpy." - ); + return module_::import((std::string("numpy.core.") + submodule_name).c_str()); + } catch (error_already_set &ex) { + if (!ex.matches(PyExc_ImportError)) + throw; + throw import_error(std::string("pybind11 couldn't import ") + submodule_name + + " from numpy."); } } } @@ -283,7 +280,7 @@ struct npy_api { }; static npy_api lookup() { - module_ m = import_numpy_core_submodule("multiarray"); + module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); npy_api api; @@ -646,11 +643,8 @@ class dtype : public object { private: static object _dtype_from_pep3118() { - module_ m = import_numpy_core_submodule("_internal"); - static PyObject *obj = m.attr("_dtype_from_pep3118") - .cast() - .release() - .ptr(); + module_ m = detail::import_numpy_core_submodule("_internal"); + static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); return reinterpret_borrow(obj); } From 45cc2a0fef7e5d4849d54e3a3ad9b30cec375f85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Tue, 26 Sep 2023 15:24:14 +0200 Subject: [PATCH 05/10] Add no-inline attribute --- include/pybind11/numpy.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 81c39ec840..aa23b5661b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,17 +120,19 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } -module_ import_numpy_core_submodule(const char *submodule_name) { +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { try { return module_::import((std::string("numpy._core.") + submodule_name).c_str()); } catch (error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) + if (!ex.matches(PyExc_ImportError)) { throw; + } try { return module_::import((std::string("numpy.core.") + submodule_name).c_str()); } catch (error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) + if (!ex.matches(PyExc_ImportError)) { throw; + } throw import_error(std::string("pybind11 couldn't import ") + submodule_name + " from numpy."); } From 2adef4b261d57c276fa249159cd69a8ea2f9ac99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Tue, 26 Sep 2023 23:25:40 +0200 Subject: [PATCH 06/10] Select submodule name based on numpy version --- include/pybind11/numpy.h | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index aa23b5661b..b9895f9305 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,22 +120,24 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } -PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { - try { - return module_::import((std::string("numpy._core.") + submodule_name).c_str()); - } catch (error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) { - throw; - } - try { - return module_::import((std::string("numpy.core.") + submodule_name).c_str()); - } catch (error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) { - throw; - } - throw import_error(std::string("pybind11 couldn't import ") + submodule_name - + " from numpy."); - } +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char * submodule_name) { + module_ numpy = module_::import("numpy"); + str version_string = numpy.attr("__version__"); + + module_ numpy_lib = module_::import("numpy.lib"); + object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); + int major_version = numpy_version.attr("major").cast(); + + /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially + became a private module. */ + if (major_version >= 2) { + return py::module_::import( + (std::string("numpy._core.") + submodule_name).c_str() + ); + } else { + return py::module_::import( + (std::string("numpy.core.") + submodule_name).c_str() + ); } } @@ -285,6 +287,9 @@ struct npy_api { module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); + if (api_ptr == nullptr) { + raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); From f2f497e7dc319b0f00826cd0e2458a7e681330e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 21:26:09 +0000 Subject: [PATCH 07/10] style: pre-commit fixes --- include/pybind11/numpy.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b9895f9305..6a6babf982 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,7 +120,7 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } -PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char * submodule_name) { +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { module_ numpy = module_::import("numpy"); str version_string = numpy.attr("__version__"); @@ -131,13 +131,9 @@ PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char * submodule_nam /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially became a private module. */ if (major_version >= 2) { - return py::module_::import( - (std::string("numpy._core.") + submodule_name).c_str() - ); + return py::module_::import((std::string("numpy._core.") + submodule_name).c_str()); } else { - return py::module_::import( - (std::string("numpy.core.") + submodule_name).c_str() - ); + return py::module_::import((std::string("numpy.core.") + submodule_name).c_str()); } } From 8aa97a6c77cdc249e999c17e942ae8e286b69268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Wed, 27 Sep 2023 00:13:58 +0200 Subject: [PATCH 08/10] Update pre-commit check --- .pre-commit-config.yaml | 2 +- include/pybind11/numpy.h | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1fb4ae11c..5f1ce5cbd3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -142,7 +142,7 @@ repos: - id: disallow-caps name: Disallow improper capitalization language: pygrep - entry: PyBind|Numpy|Cmake|CCache|PyTest + entry: PyBind|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ # PyLint has native support - not always usable, but works for us diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 75981a37dc..a5b36b6223 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -130,11 +130,13 @@ PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially became a private module. */ + std::string numpy_core_path; if (major_version >= 2) { - return module_::import((std::string("numpy._core.") + submodule_name).c_str()); + numpy_core_path = std::string("numpy._core"); } else { - return module_::import((std::string("numpy.core.") + submodule_name).c_str()); + numpy_core_path = std::string("numpy.core"); } + return module_::import((numpy_core_path + "." + submodule_name).c_str()); } template From c656925763936c91333be4e1fe130842913c7d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= Date: Wed, 27 Sep 2023 00:39:33 +0200 Subject: [PATCH 09/10] Add error_already_set and simplify if statement --- include/pybind11/numpy.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a5b36b6223..23c38660e9 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -130,12 +130,7 @@ PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially became a private module. */ - std::string numpy_core_path; - if (major_version >= 2) { - numpy_core_path = std::string("numpy._core"); - } else { - numpy_core_path = std::string("numpy.core"); - } + std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core"; return module_::import((numpy_core_path + "." + submodule_name).c_str()); } @@ -287,6 +282,7 @@ struct npy_api { void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); if (api_ptr == nullptr) { raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + throw error_already_set(); } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; From a89e0a64486e0a159e0639432e4600673631a104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= <8431159+mtsokol@users.noreply.github.com> Date: Wed, 27 Sep 2023 10:10:02 +0200 Subject: [PATCH 10/10] Update .pre-commit-config.yaml Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f1ce5cbd3..c511ab916a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -142,7 +142,7 @@ repos: - id: disallow-caps name: Disallow improper capitalization language: pygrep - entry: PyBind|Cmake|CCache|PyTest + entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ # PyLint has native support - not always usable, but works for us