build(python): Update pyo3 and numpy crates to version 0.23 (#2…

…0111)

.gitignore

@@ -1,6 +1,7 @@
 *.iml
 *.so
 *.pyd
+*.pdb
 *.ipynb
 .ENV
 .env

Cargo.toml

@@ -56,13 +56,13 @@ memmap = { package = "memmap2", version = "0.9" }
 multiversion = "0.7"
 ndarray = { version = "0.16", default-features = false }
 num-traits = "0.2"
-numpy = "0.22"
+numpy = "0.23"
 object_store = { version = "0.11", default-features = false }
 once_cell = "1"
 parking_lot = "0.12"
 percent-encoding = "2.3"
 pin-project-lite = "0.2"
-pyo3 = "0.22"
+pyo3 = { git = "https://github.com/bschoenmaeckers/pyo3.git", branch = "release-0.23" }
 rand = "0.8"
 rand_distr = "0.4"
 raw-cpuid = "11"
@@ -136,6 +136,8 @@ features = [
 [patch.crates-io]
 # packed_simd_2 = { git = "https://github.com/rust-lang/packed_simd", rev = "e57c7ba11386147e6d2cbad7c88f376aab4bdc86" }
 # simd-json = { git = "https://github.com/ritchie46/simd-json", branch = "alignment" }
+pyo3 = { git = "https://github.com/bschoenmaeckers/pyo3.git", branch = "release-0.23" }
+pyo3-ffi = { git = "https://github.com/bschoenmaeckers/pyo3.git", branch = "release-0.23" }
 
 [profile.mindebug-dev]
 inherits = "dev"

crates/polars-error/src/lib.rs

@@ -3,6 +3,7 @@ mod warning;
 
 use std::borrow::Cow;
 use std::collections::TryReserveError;
+use std::convert::Infallible;
 use std::error::Error;
 use std::fmt::{self, Display, Formatter, Write};
 use std::ops::Deref;
@@ -168,6 +169,12 @@ impl From<TryReserveError> for PolarsError {
     }
 }
 
+impl From<Infallible> for PolarsError {
+    fn from(_: Infallible) -> Self {
+        unreachable!()
+    }
+}
+
 pub type PolarsResult<T> = Result<T, PolarsError>;
 
 impl PolarsError {

crates/polars-mem-engine/src/executors/scan/python_scan.rs

@@ -2,8 +2,8 @@ use polars_core::error::to_compute_err;
 use polars_core::utils::accumulate_dataframes_vertical;
 use pyo3::exceptions::PyStopIteration;
 use pyo3::prelude::*;
-use pyo3::types::PyBytes;
-use pyo3::{intern, PyTypeInfo};
+use pyo3::types::{PyBytes, PyNone};
+use pyo3::{intern, IntoPyObjectExt, PyTypeInfo};
 
 use super::*;
 
@@ -41,7 +41,7 @@ impl Executor for PythonScanExec {
         let with_columns = self.options.with_columns.take();
         let n_rows = self.options.n_rows.take();
         Python::with_gil(|py| {
-            let pl = PyModule::import_bound(py, intern!(py, "polars")).unwrap();
+            let pl = PyModule::import(py, intern!(py, "polars")).unwrap();
             let utils = pl.getattr(intern!(py, "_utils")).unwrap();
             let callable = utils.getattr(intern!(py, "_execute_from_rust")).unwrap();
 
@@ -50,14 +50,14 @@ impl Executor for PythonScanExec {
             let with_columns = with_columns.map(|cols| cols.iter().cloned().collect::<Vec<_>>());
 
             let predicate = match &self.options.predicate {
-                PythonPredicate::PyArrow(s) => s.into_py(py),
-                PythonPredicate::None => None::<()>.into_py(py),
+                PythonPredicate::PyArrow(s) => s.into_bound_py_any(py).unwrap(),
+                PythonPredicate::None => None::<()>.into_bound_py_any(py).unwrap(),
                 PythonPredicate::Polars(_) => {
                     assert!(self.predicate.is_some(), "should be set");
 
                     match &self.predicate_serialized {
-                        None => None::<()>.into_py(py),
-                        Some(buf) => PyBytes::new_bound(py, buf).to_object(py),
+                        None => PyNone::get(py).to_owned().into_any(),
+                        Some(buf) => PyBytes::new(py, buf).into_any(),
                     }
                 },
             };
@@ -125,7 +125,7 @@ impl Executor for PythonScanExec {
                         }
                         chunks.push(df)
                     },
-                    Err(err) if err.matches(py, PyStopIteration::type_object_bound(py)) => break,
+                    Err(err) if err.matches(py, PyStopIteration::type_object(py))? => break,
                     Err(err) => {
                         polars_bail!(ComputeError: "caught exception during execution of a Python source, exception: {}", err)
                     },

crates/polars-plan/src/dsl/python_udf.rs

@@ -78,11 +78,11 @@ impl PythonUdfExpression {
 
         // Load UDF
         Python::with_gil(|py| {
-            let pickle = PyModule::import_bound(py, "pickle")
+            let pickle = PyModule::import(py, "pickle")
                 .expect("unable to import 'pickle'")
                 .getattr("loads")
                 .unwrap();
-            let arg = (PyBytes::new_bound(py, remainder),);
+            let arg = (PyBytes::new(py, remainder),);
             let python_function = pickle.call1(arg).map_err(from_pyerr)?;
             Ok(Arc::new(Self::new(
                 python_function.into(),
@@ -136,15 +136,15 @@ impl ColumnsUdf for PythonUdfExpression {
 
         Python::with_gil(|py| {
             // Try pickle to serialize the UDF, otherwise fall back to cloudpickle.
-            let pickle = PyModule::import_bound(py, "pickle")
+            let pickle = PyModule::import(py, "pickle")
                 .expect("unable to import 'pickle'")
                 .getattr("dumps")
                 .unwrap();
             let pickle_result = pickle.call1((self.python_function.clone_ref(py),));
             let (dumped, use_cloudpickle) = match pickle_result {
                 Ok(dumped) => (dumped, false),
                 Err(_) => {
-                    let cloudpickle = PyModule::import_bound(py, "cloudpickle")
+                    let cloudpickle = PyModule::import(py, "cloudpickle")
                         .map_err(from_pyerr)?
                         .getattr("dumps")
                         .unwrap();

crates/polars-python/src/cloud.rs

@@ -14,11 +14,11 @@ use crate::lazyframe::visit::NodeTraverser;
 use crate::{PyDataFrame, PyLazyFrame};
 
 #[pyfunction]
-pub fn prepare_cloud_plan(lf: PyLazyFrame, py: Python) -> PyResult<PyObject> {
+pub fn prepare_cloud_plan(lf: PyLazyFrame, py: Python<'_>) -> PyResult<Bound<'_, PyBytes>> {
     let plan = lf.ldf.logical_plan;
     let bytes = polars::prelude::prepare_cloud_plan(plan).map_err(PyPolarsErr::from)?;
 
-    Ok(PyBytes::new_bound(py, &bytes).to_object(py))
+    Ok(PyBytes::new(py, &bytes))
 }
 
 /// Take a serialized `IRPlan` and execute it on the GPU engine.
@@ -62,13 +62,13 @@ fn gpu_post_opt(
     expr_arena: &mut Arena<AExpr>,
 ) -> PolarsResult<()> {
     // Get cuDF Python function.
-    let cudf = PyModule::import_bound(py, intern!(py, "cudf_polars")).unwrap();
+    let cudf = PyModule::import(py, intern!(py, "cudf_polars")).unwrap();
     let lambda = cudf.getattr(intern!(py, "execute_with_cudf")).unwrap();
 
     // Define cuDF config.
-    let polars = PyModule::import_bound(py, intern!(py, "polars")).unwrap();
+    let polars = PyModule::import(py, intern!(py, "polars")).unwrap();
     let engine = polars.getattr(intern!(py, "GPUEngine")).unwrap();
-    let kwargs = [("raise_on_fail", true)].into_py_dict_bound(py);
+    let kwargs = [("raise_on_fail", true)].into_py_dict(py).unwrap();
     let engine = engine.call((), Some(&kwargs)).unwrap();
 
     // Define node traverser.
@@ -79,7 +79,7 @@ fn gpu_post_opt(
 
     // Pass the node visitor which allows the Python callback to replace parts of the query plan.
     // Remove "cuda" or specify better once we have multiple post-opt callbacks.
-    let kwargs = [("config", engine)].into_py_dict_bound(py);
+    let kwargs = [("config", engine)].into_py_dict(py).unwrap();
     lambda
         .call((nt,), Some(&kwargs))
         .map_err(|e| polars_err!(ComputeError: "'cuda' conversion failed: {}", e))?;