Trim down arrow dependency

pyvortex/Cargo.toml

@@ -19,7 +19,7 @@ name = "pyvortex"
 crate-type = ["rlib", "cdylib"]
 
 [dependencies]
-arrow = { version = "50.0.0", features = ["ffi"] }
+arrow = { version = "50.0.0", features = ["pyarrow"] }
 vortex-array = { path = "../vortex-array" }
 vortex-alp = { path = "../vortex-alp" }
 vortex-dict = { path = "../vortex-dict" }

vortex-array/Cargo.toml

@@ -20,7 +20,12 @@ workspace = true
 
 [dependencies]
 allocator-api2 = "0.2.16"
-arrow = { version = "50.0.0", features = ["pyarrow"] }
+arrow-array = { version = "50.0.0" }
+arrow-buffer = { version = "50.0.0" }
+arrow-schema = { version = "50.0.0" }
+arrow-data = { version = "50.0.0" }
+arrow-cast = { version = "50.0.0" }
+arrow-select = { version = "50.0.0" }
 dyn-clone = "1.0.16"
 half = "2.3.1"
 humansize = "2.1.3"

vortex-array/src/array/bool/compute.rs

@@ -1,4 +1,4 @@
-use arrow::buffer::BooleanBuffer;
+use arrow_buffer::buffer::BooleanBuffer;
 use itertools::Itertools;
 
 use crate::array::bool::BoolArray;

vortex-array/src/array/bool/mod.rs

@@ -2,8 +2,9 @@ use std::any::Any;
 use std::iter;
 use std::sync::{Arc, RwLock};
 
-use arrow::array::{ArrayRef as ArrowArrayRef, AsArray, BooleanArray};
-use arrow::buffer::{BooleanBuffer, NullBuffer};
+use arrow_array::array::{ArrayRef as ArrowArrayRef, BooleanArray};
+use arrow_array::cast::AsArray;
+use arrow_buffer::buffer::{BooleanBuffer, NullBuffer};
 use linkme::distributed_slice;
 
 use crate::arrow::CombineChunks;

vortex-array/src/array/bool/serde.rs

@@ -1,6 +1,6 @@
 use std::io;
 
-use arrow::buffer::BooleanBuffer;
+use arrow_buffer::buffer::BooleanBuffer;
 
 use crate::array::bool::{BoolArray, BoolEncoding};
 use crate::array::{Array, ArrayRef};

vortex-array/src/array/chunked/mod.rs

@@ -2,7 +2,7 @@ use std::any::Any;
 use std::sync::{Arc, RwLock};
 use std::vec::IntoIter;
 
-use arrow::array::ArrayRef as ArrowArrayRef;
+use arrow_array::array::ArrayRef as ArrowArrayRef;
 use itertools::Itertools;
 use linkme::distributed_slice;
 
@@ -252,10 +252,10 @@ impl Iterator for ChunkedArrowIterator {
 
 #[cfg(test)]
 mod test {
-    use arrow::array::cast::AsArray;
-    use arrow::array::types::UInt64Type;
-    use arrow::array::ArrayRef as ArrowArrayRef;
-    use arrow::array::ArrowPrimitiveType;
+    use arrow_array::array::ArrayRef as ArrowArrayRef;
+    use arrow_array::array::ArrowPrimitiveType;
+    use arrow_array::cast::AsArray;
+    use arrow_array::types::UInt64Type;
     use itertools::Itertools;
 
     use crate::array::chunked::ChunkedArray;

vortex-array/src/array/mod.rs

@@ -1,7 +1,7 @@
 use std::any::Any;
 use std::fmt::{Debug, Display, Formatter};
 
-use arrow::array::ArrayRef as ArrowArrayRef;
+use arrow_array::array::ArrayRef as ArrowArrayRef;
 use linkme::distributed_slice;
 
 use crate::array::bool::{BoolArray, BoolEncoding};

vortex-array/src/array/primitive/mod.rs

@@ -7,8 +7,10 @@ use std::ptr::NonNull;
 use std::sync::{Arc, RwLock};
 
 use allocator_api2::alloc::Allocator;
-use arrow::array::{make_array, ArrayData, AsArray};
-use arrow::buffer::{Buffer, NullBuffer, ScalarBuffer};
+use arrow_array::array::make_array;
+use arrow_array::cast::AsArray;
+use arrow_buffer::buffer::{Buffer, NullBuffer, ScalarBuffer};
+use arrow_data::ArrayData;
 use linkme::distributed_slice;
 
 use crate::array::bool::BoolArray;

vortex-array/src/array/primitive/stats.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::mem::size_of;
 
-use arrow::buffer::BooleanBuffer;
+use arrow_buffer::buffer::BooleanBuffer;
 
 use crate::array::primitive::PrimitiveArray;
 use crate::compute::cast::cast_bool;

vortex-array/src/array/sparse/mod.rs

@@ -2,12 +2,11 @@ use std::any::Any;
 use std::iter;
 use std::sync::{Arc, RwLock};
 
-use arrow::array::AsArray;
-use arrow::array::{
-    ArrayRef as ArrowArrayRef, BooleanBufferBuilder, PrimitiveArray as ArrowPrimitiveArray,
-};
-use arrow::buffer::{NullBuffer, ScalarBuffer};
-use arrow::datatypes::UInt64Type;
+use arrow_array::array::{ArrayRef as ArrowArrayRef, PrimitiveArray as ArrowPrimitiveArray};
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt64Type;
+use arrow_buffer::buffer::{NullBuffer, ScalarBuffer};
+use arrow_buffer::BooleanBufferBuilder;
 use linkme::distributed_slice;
 
 use crate::array::ENCODINGS;
@@ -85,7 +84,7 @@ impl SparseArray {
         let mut indices = Vec::with_capacity(self.len());
         self.indices().iter_arrow().for_each(|c| {
             indices.extend(
-                arrow::compute::cast(c.as_ref(), &arrow::datatypes::DataType::UInt64)
+                arrow_cast::cast(c.as_ref(), &arrow_schema::DataType::UInt64)
                     .unwrap()
                     .as_primitive::<UInt64Type>()
                     .values()
@@ -231,8 +230,8 @@ impl Encoding for SparseEncoding {
 
 #[cfg(test)]
 mod test {
-    use arrow::array::AsArray;
-    use arrow::datatypes::Int32Type;
+    use arrow_array::cast::AsArray;
+    use arrow_array::types::Int32Type;
     use itertools::Itertools;
 
     use crate::array::sparse::SparseArray;

vortex-array/src/array/struct_/mod.rs

@@ -1,12 +1,13 @@
 use std::any::Any;
 use std::sync::{Arc, RwLock};
 
-use arrow::array::StructArray as ArrowStructArray;
-use arrow::array::{Array as ArrowArray, ArrayRef as ArrowArrayRef};
-use arrow::datatypes::{Field, Fields};
 use itertools::Itertools;
 use linkme::distributed_slice;
 
+use arrow_array::array::StructArray as ArrowStructArray;
+use arrow_array::array::{Array as ArrowArray, ArrayRef as ArrowArrayRef};
+use arrow_schema::{Field, Fields};
+
 use crate::arrow::aligned_iter::AlignedArrowArrayIterator;
 use crate::compress::EncodingCompression;
 use crate::dtype::{DType, FieldNames};
@@ -208,10 +209,10 @@ impl ArrayDisplay for StructArray {
 mod test {
     use std::sync::Arc;
 
-    use arrow::array::types::UInt64Type;
-    use arrow::array::PrimitiveArray as ArrowPrimitiveArray;
-    use arrow::array::StructArray as ArrowStructArray;
-    use arrow::array::{Array as ArrowArray, GenericStringArray as ArrowStringArray};
+    use arrow_array::array::PrimitiveArray as ArrowPrimitiveArray;
+    use arrow_array::array::StructArray as ArrowStructArray;
+    use arrow_array::array::{Array as ArrowArray, GenericStringArray as ArrowStringArray};
+    use arrow_array::types::UInt64Type;
 
     use crate::array::struct_::StructArray;
     use crate::array::Array;

vortex-array/src/array/typed/mod.rs

@@ -1,7 +1,7 @@
 use std::any::Any;
 use std::sync::{Arc, RwLock};
 
-use arrow::datatypes::DataType;
+use arrow_schema::DataType;
 use linkme::distributed_slice;
 
 use crate::array::{Array, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, ENCODINGS};
@@ -88,9 +88,9 @@ impl Array for TypedArray {
     fn iter_arrow(&self) -> Box<ArrowIterator> {
         let datatype: DataType = self.dtype().into();
         Box::new(
-            self.array.iter_arrow().map(move |arr| {
-                arrow::compute::kernels::cast::cast(arr.as_ref(), &datatype).unwrap()
-            }),
+            self.array
+                .iter_arrow()
+                .map(move |arr| arrow_cast::cast(arr.as_ref(), &datatype).unwrap()),
         )
     }
 
@@ -153,11 +153,11 @@ impl ArrayDisplay for TypedArray {
 
 #[cfg(test)]
 mod test {
+    use arrow_array::cast::AsArray;
+    use arrow_array::types::Time64MicrosecondType;
+    use arrow_array::Time64MicrosecondArray;
     use std::iter;
 
-    use arrow::array::cast::AsArray;
-    use arrow::array::types::Time64MicrosecondType;
-    use arrow::array::Time64MicrosecondArray;
     use itertools::Itertools;
 
     use crate::array::typed::TypedArray;

vortex-array/src/array/varbin/mod.rs

@@ -2,9 +2,11 @@ use std::any::Any;
 use std::iter;
 use std::sync::{Arc, RwLock};
 
-use arrow::array::{make_array, Array as ArrowArray, ArrayData, AsArray};
-use arrow::buffer::NullBuffer;
-use arrow::datatypes::UInt8Type;
+use arrow_array::array::{make_array, Array as ArrowArray};
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt8Type;
+use arrow_buffer::buffer::NullBuffer;
+use arrow_data::ArrayData;
 use linkme::distributed_slice;
 use num_traits::{FromPrimitive, Unsigned};
 
@@ -392,7 +394,8 @@ impl<'a> FromIterator<Option<&'a str>> for VarBinArray {
 
 #[cfg(test)]
 mod test {
-    use arrow::array::{AsArray, GenericStringArray as ArrowStringArray};
+    use arrow_array::array::GenericStringArray as ArrowStringArray;
+    use arrow_array::cast::AsArray;
 
     use crate::array::primitive::PrimitiveArray;
     use crate::array::varbin::VarBinArray;

vortex-array/src/array/varbin/values_iter.rs

@@ -1,11 +1,10 @@
-use arrow::array::AsArray;
-use arrow::datatypes::UInt8Type;
-
 use crate::array::primitive::PrimitiveArray;
 use crate::array::Array;
 use crate::arrow::CombineChunks;
 use crate::compute::scalar_at::scalar_at;
 use crate::match_each_native_ptype;
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt8Type;
 use num_traits::AsPrimitive;
 
 #[derive(Debug)]

vortex-array/src/array/varbinview/mod.rs

@@ -6,9 +6,10 @@ use std::str::from_utf8_unchecked;
 use std::sync::{Arc, RwLock};
 use std::{iter, mem};
 
-use arrow::array::cast::AsArray;
-use arrow::array::types::UInt8Type;
-use arrow::array::{ArrayRef as ArrowArrayRef, BinaryBuilder, StringBuilder};
+use arrow_array::array::ArrayRef as ArrowArrayRef;
+use arrow_array::builder::{BinaryBuilder, StringBuilder};
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt8Type;
 use linkme::distributed_slice;
 
 use crate::array::{
@@ -343,7 +344,7 @@ impl ArrayDisplay for VarBinViewArray {
 
 #[cfg(test)]
 mod test {
-    use arrow::array::GenericStringArray as ArrowStringArray;
+    use arrow_array::array::GenericStringArray as ArrowStringArray;
 
     use crate::array::primitive::PrimitiveArray;
 

vortex-array/src/arrow/aligned_iter.rs

@@ -1,4 +1,4 @@
-use arrow::array::{Array as ArrowArray, ArrayRef};
+use arrow_array::array::{Array as ArrowArray, ArrayRef};
 
 pub struct AlignedArray {
     iter: Box<dyn Iterator<Item = ArrayRef>>,

vortex-array/src/arrow/convert.rs

@@ -1,9 +1,8 @@
+use arrow_array::{RecordBatch, RecordBatchReader};
 use std::iter::zip;
 use std::sync::Arc;
 
-use arrow::array::RecordBatch;
-use arrow::array::RecordBatchReader;
-use arrow::datatypes::{
+use arrow_schema::{
     DataType, Field, FieldRef, Fields, Schema, SchemaRef, TimeUnit as ArrowTimeUnit,
 };
 use itertools::Itertools;

vortex-array/src/arrow/mod.rs

@@ -1,4 +1,5 @@
-use arrow::array::ArrayRef;
+use arrow_array::array::ArrayRef;
+use arrow_select::concat::concat;
 use itertools::Itertools;
 
 use crate::array::ArrowIterator;
@@ -14,7 +15,7 @@ impl CombineChunks for Box<ArrowIterator> {
     fn combine_chunks(self) -> ArrayRef {
         let chunks = self.collect_vec();
         let chunk_refs = chunks.iter().map(|a| a.as_ref()).collect_vec();
-        arrow::compute::concat(&chunk_refs).unwrap()
+        concat(&chunk_refs).unwrap()
     }
 }
 
@@ -26,7 +27,7 @@ macro_rules! match_arrow_numeric_type {
         use $crate::dtype::IntWidth::*;
         use $crate::dtype::Signedness::*;
         use $crate::dtype::FloatWidth;
-        use arrow::datatypes::*;
+        use arrow_array::types::*;
         match $self {
             Int(_8, Unsigned, _) => __with__! {UInt8Type},
             Int(_16, Unsigned, _) => __with__!{UInt16Type},

vortex-array/src/encode.rs

@@ -1,24 +1,24 @@
 use std::sync::Arc;
 
-use arrow::array::cast::AsArray;
-use arrow::array::types::{
-    Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type,
-    UInt32Type, UInt64Type, UInt8Type,
+use arrow_array::array::{
+    Array as ArrowArray, ArrayRef as ArrowArrayRef, BooleanArray as ArrowBooleanArray,
+    GenericByteArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray,
+    StructArray as ArrowStructArray,
 };
-use arrow::array::{
-    as_null_array, Array as ArrowArray, ArrayRef as ArrowArrayRef,
-    BooleanArray as ArrowBooleanArray, GenericByteArray, NullArray as ArrowNullArray,
-    PrimitiveArray as ArrowPrimitiveArray, StructArray as ArrowStructArray,
+use arrow_array::array::{ArrowPrimitiveType, OffsetSizeTrait};
+use arrow_array::cast::{as_null_array, AsArray};
+use arrow_array::types::{
+    ByteArrayType, Date32Type, Date64Type, DurationMicrosecondType, DurationMillisecondType,
+    DurationNanosecondType, DurationSecondType, Time32MillisecondType, Time32SecondType,
+    Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,
+    TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
 };
-use arrow::array::{ArrowPrimitiveType, OffsetSizeTrait};
-use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};
-use arrow::datatypes::{
-    ByteArrayType, DataType, Date32Type, Date64Type, DurationMicrosecondType,
-    DurationMillisecondType, DurationNanosecondType, DurationSecondType, Time32MillisecondType,
-    Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit,
-    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
-    TimestampSecondType,
+use arrow_array::types::{
+    Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type,
+    UInt32Type, UInt64Type, UInt8Type,
 };
+use arrow_buffer::buffer::{Buffer, NullBuffer, OffsetBuffer};
+use arrow_schema::{DataType, TimeUnit};
 
 use crate::array::bool::BoolArray;
 use crate::array::constant::ConstantArray;