docs(python): Add examples for polars.read_parquet and polars.read_pa…

…rquet_schema

.github/workflows/codecov.yml

@@ -70,6 +70,7 @@ jobs:
         run: >
           cargo test --all-features
           -p polars-arrow
+          -p polars-compute
           -p polars-core
           -p polars-io
           -p polars-lazy

.github/workflows/test-rust.yml

@@ -46,6 +46,7 @@ jobs:
         run: >
           cargo test --all-features --no-run
           -p polars-arrow
+          -p polars-compute
           -p polars-core
           -p polars-io
           -p polars-lazy
@@ -61,6 +62,7 @@ jobs:
         run: >
           cargo test --all-features
           -p polars-arrow
+          -p polars-compute
           -p polars-core
           -p polars-io
           -p polars-lazy

crates/Makefile

@@ -42,6 +42,7 @@ miri:  ## Run miri
 .PHONY: test
 test:  ## Run tests
 	cargo test --all-features \
+		-p polars-compute \
 		-p polars-core \
 		-p polars-io \
 		-p polars-lazy \
@@ -57,6 +58,7 @@ test:  ## Run tests
 .PHONY: nextest
 nextest:  ## Run tests with nextest
 	cargo nextest run --all-features \
+		-p polars-compute \
 		-p polars-core \
 		-p polars-io \
 		-p polars-lazy \

crates/polars-arrow/Cargo.toml

@@ -159,7 +159,9 @@ compute = [
 simd = []
 
 # polars-arrow
-timezones = []
+timezones = [
+  "chrono-tz",
+]
 dtype-array = []
 dtype-decimal = ["atoi", "itoap"]
 bigidx = []

crates/polars-arrow/src/array/primitive/mutable.rs

@@ -127,17 +127,20 @@ impl<T: NativeType> MutablePrimitiveArray<T> {
         }
     }
 
+    #[inline]
+    pub fn push_value(&mut self, value: T) {
+        self.values.push(value);
+        match &mut self.validity {
+            Some(validity) => validity.push(true),
+            None => {},
+        }
+    }
+
     /// Adds a new value to the array.
     #[inline]
     pub fn push(&mut self, value: Option<T>) {
         match value {
-            Some(value) => {
-                self.values.push(value);
-                match &mut self.validity {
-                    Some(validity) => validity.push(true),
-                    None => {},
-                }
-            },
+            Some(value) => self.push_value(value),
             None => {
                 self.values.push(T::default());
                 match &mut self.validity {
@@ -288,6 +291,24 @@ impl<T: NativeType> MutablePrimitiveArray<T> {
     pub fn freeze(self) -> PrimitiveArray<T> {
         self.into()
     }
+
+    /// Clears the array, removing all values.
+    ///
+    /// Note that this method has no effect on the allocated capacity
+    /// of the array.
+    pub fn clear(&mut self) {
+        self.values.clear();
+        self.validity = None;
+    }
+
+    /// Apply a function that temporarily freezes this `MutableArray` into a `PrimitiveArray`.
+    pub fn with_freeze<K, F: FnOnce(&PrimitiveArray<T>) -> K>(&mut self, f: F) -> K {
+        let mutable = std::mem::take(self);
+        let arr = mutable.freeze();
+        let out = f(&arr);
+        *self = arr.into_mut().right().unwrap();
+        out
+    }
 }
 
 /// Accessors

crates/polars-arrow/src/bitmap/immutable.rs

@@ -11,6 +11,7 @@ use crate::bitmap::iterator::{
     FastU32BitmapIter, FastU56BitmapIter, FastU64BitmapIter, TrueIdxIter,
 };
 use crate::buffer::Bytes;
+use crate::legacy::utils::FromTrustedLenIterator;
 use crate::trusted_len::TrustedLen;
 
 const UNKNOWN_BIT_COUNT: u64 = u64::MAX;
@@ -484,6 +485,15 @@ impl FromIterator<bool> for Bitmap {
     }
 }
 
+impl FromTrustedLenIterator<bool> for Bitmap {
+    fn from_iter_trusted_length<T: IntoIterator<Item = bool>>(iter: T) -> Self
+    where
+        T::IntoIter: TrustedLen,
+    {
+        MutableBitmap::from_trusted_len_iter(iter.into_iter()).into()
+    }
+}
+
 impl Bitmap {
     /// Creates a new [`Bitmap`] from an iterator of booleans.
     ///

crates/polars-arrow/src/compute/cast/mod.rs

@@ -17,6 +17,7 @@ pub use boolean_to::*;
 pub use decimal_to::*;
 pub use dictionary_to::*;
 use polars_error::{polars_bail, polars_ensure, polars_err, PolarsResult};
+use polars_utils::IdxSize;
 pub use primitive_to::*;
 pub use utf8_to::*;
 
@@ -26,7 +27,6 @@ use crate::compute::cast::binview_to::{
     utf8view_to_naive_timestamp_dyn, view_to_binary,
 };
 use crate::datatypes::*;
-use crate::legacy::index::IdxSize;
 use crate::match_integer_type;
 use crate::offset::{Offset, Offsets};
 use crate::temporal_conversions::utf8view_to_timestamp;

crates/polars-arrow/src/compute/take/bitmap.rs

@@ -1,5 +1,6 @@
+use polars_utils::IdxSize;
+
 use crate::bitmap::Bitmap;
-use crate::legacy::index::IdxSize;
 
 /// # Safety
 /// doesn't do any bound checks

crates/polars-arrow/src/compute/take/boolean.rs

@@ -1,7 +1,8 @@
+use polars_utils::IdxSize;
+
 use super::bitmap::take_bitmap_unchecked;
 use crate::array::{Array, BooleanArray, PrimitiveArray};
 use crate::bitmap::{Bitmap, MutableBitmap};
-use crate::legacy::index::IdxSize;
 
 // take implementation when neither values nor indices contain nulls
 unsafe fn take_no_validity(values: &Bitmap, indices: &[IdxSize]) -> (Bitmap, Option<Bitmap>) {

crates/polars-arrow/src/legacy/index.rs

@@ -1,9 +1,7 @@
 use num_traits::{NumCast, Signed, Zero};
+use polars_utils::IdxSize;
 
-#[cfg(not(feature = "bigidx"))]
-use crate::array::UInt32Array;
-#[cfg(feature = "bigidx")]
-use crate::array::UInt64Array;
+use crate::array::PrimitiveArray;
 
 pub trait IndexToUsize {
     /// Translate the negative index to an offset.
@@ -33,17 +31,8 @@ where
     }
 }
 
-/// The type used by polars to index data.
-#[cfg(not(feature = "bigidx"))]
-pub type IdxSize = u32;
-#[cfg(feature = "bigidx")]
-pub type IdxSize = u64;
-
-#[cfg(not(feature = "bigidx"))]
-pub type IdxArr = UInt32Array;
-#[cfg(feature = "bigidx")]
-pub type IdxArr = UInt64Array;
-
 pub fn indexes_to_usizes(idx: &[IdxSize]) -> impl Iterator<Item = usize> + '_ {
     idx.iter().map(|idx| *idx as usize)
 }
+
+pub type IdxArr = PrimitiveArray<IdxSize>;

crates/polars-arrow/src/legacy/kernels/fixed_size_list.rs

@@ -1,3 +1,5 @@
+use polars_utils::IdxSize;
+
 use crate::array::{ArrayRef, FixedSizeListArray, PrimitiveArray};
 use crate::compute::take::take_unchecked;
 use crate::legacy::prelude::*;

crates/polars-arrow/src/legacy/kernels/list.rs

@@ -1,3 +1,5 @@
+use polars_utils::IdxSize;
+
 use crate::array::{ArrayRef, ListArray};
 use crate::compute::take::take_unchecked;
 use crate::legacy::prelude::*;

crates/polars-arrow/src/legacy/kernels/set.rs

@@ -1,12 +1,12 @@
 use std::ops::BitOr;
 
 use polars_error::polars_err;
+use polars_utils::IdxSize;
 
 use crate::array::*;
 use crate::datatypes::ArrowDataType;
 use crate::legacy::array::default_arrays::FromData;
 use crate::legacy::error::PolarsResult;
-use crate::legacy::index::IdxSize;
 use crate::legacy::kernels::BinaryMaskedSliceIterator;
 use crate::legacy::trusted_len::TrustedLenPush;
 use crate::types::NativeType;

crates/polars-arrow/src/legacy/kernels/sort_partition.rs

@@ -1,6 +1,7 @@
 use std::fmt::Debug;
 
-use crate::legacy::index::IdxSize;
+use polars_utils::IdxSize;
+
 use crate::types::NativeType;
 
 /// Find partition indexes such that every partition contains unique groups.