Vortex Buffer Crate (#299)

spiraldb · May 6, 2024 · 19ad296 · 19ad296
1 parent bfe8a3c
commit 19ad296
Show file tree

Hide file tree

Showing 19 changed files with 54 additions and 28 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -6,6 +6,7 @@ members = [
     "pyvortex",
     "vortex-alp",
     "vortex-array",
+    "vortex-buffer",
     "vortex-datetime-parts",
     "vortex-dict",
     "vortex-error",

diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml
@@ -33,6 +33,7 @@ num-traits = { workspace = true }
 num_enum = { workspace = true }
 paste = { workspace = true }
 rand = { workspace = true }
+vortex-buffer = { path = "../vortex-buffer" }
 vortex-dtype = { path = "../vortex-dtype", features = ["serde"] }
 vortex-error = { path = "../vortex-error", features = ["flexbuffers"] }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }

diff --git a/vortex-array/src/array/bool/mod.rs b/vortex-array/src/array/bool/mod.rs
@@ -1,8 +1,8 @@
 use arrow_buffer::BooleanBuffer;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
+use vortex_buffer::Buffer;
 
-use crate::buffer::Buffer;
 use crate::validity::{ArrayValidity, ValidityMetadata};
 use crate::validity::{LogicalValidity, Validity};
 use crate::visitor::{AcceptArrayVisitor, ArrayVisitor};

diff --git a/vortex-array/src/array/primitive/mod.rs b/vortex-array/src/array/primitive/mod.rs
@@ -2,10 +2,10 @@ use arrow_buffer::{ArrowNativeType, ScalarBuffer};
 use itertools::Itertools;
 use num_traits::AsPrimitive;
 use serde::{Deserialize, Serialize};
+use vortex_buffer::Buffer;
 use vortex_dtype::{match_each_native_ptype, NativePType, PType};
 use vortex_error::vortex_bail;
 
-use crate::buffer::Buffer;
 use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata};
 use crate::visitor::{AcceptArrayVisitor, ArrayVisitor};
 use crate::ArrayFlatten;

diff --git a/vortex-array/src/data.rs b/vortex-array/src/data.rs
@@ -1,10 +1,10 @@
 use std::sync::{Arc, RwLock};
 
+use vortex_buffer::{Buffer, OwnedBuffer};
 use vortex_dtype::DType;
 use vortex_error::{vortex_err, VortexResult};
 use vortex_scalar::Scalar;
 
-use crate::buffer::{Buffer, OwnedBuffer};
 use crate::encoding::EncodingRef;
 use crate::stats::{Stat, Statistics, StatsSet};
 use crate::{Array, ArrayMetadata, IntoArray, OwnedArray, ToArray};

diff --git a/vortex-array/src/implementation.rs b/vortex-array/src/implementation.rs
@@ -1,14 +1,14 @@
+use vortex_buffer::{Buffer, OwnedBuffer};
 use vortex_dtype::DType;
 use vortex_error::{vortex_bail, VortexError, VortexResult};
 
-use crate::buffer::{Buffer, OwnedBuffer};
 use crate::encoding::{ArrayEncoding, ArrayEncodingRef, EncodingRef};
 use crate::encoding::{ArrayEncodingExt, EncodingId};
 use crate::stats::{ArrayStatistics, Statistics};
 use crate::visitor::ArrayVisitor;
 use crate::{
     Array, ArrayDType, ArrayData, ArrayMetadata, AsArray, GetArrayMetadata, IntoArray,
-    IntoArrayData, ToArrayData, ToStatic,
+    IntoArrayData, ToArrayData,
 };
 use crate::{ArrayTrait, TryDeserializeArrayMetadata};
 

diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs
@@ -1,7 +1,6 @@
 pub mod accessor;
 pub mod array;
 pub mod arrow;
-pub mod buffer;
 pub mod compress;
 pub mod compute;
 mod context;
@@ -29,10 +28,10 @@ pub use implementation::*;
 pub use metadata::*;
 pub use typed::*;
 pub use view::*;
+use vortex_buffer::Buffer;
 use vortex_dtype::DType;
 use vortex_error::VortexResult;
 
-use crate::buffer::Buffer;
 use crate::compute::ArrayCompute;
 use crate::encoding::{ArrayEncodingRef, EncodingRef};
 use crate::stats::{ArrayStatistics, ArrayStatisticsCompute};

diff --git a/vortex-array/src/tree.rs b/vortex-array/src/tree.rs
@@ -2,10 +2,10 @@ use std::fmt;
 
 use humansize::{format_size, DECIMAL};
 use serde::ser::Error;
+use vortex_buffer::Buffer;
 use vortex_error::{VortexError, VortexResult};
 
 use crate::array::chunked::ChunkedArray;
-use crate::buffer::Buffer;
 use crate::visitor::ArrayVisitor;
 use crate::{Array, ToArrayData};
 

diff --git a/vortex-array/src/typed.rs b/vortex-array/src/typed.rs
@@ -1,9 +1,9 @@
 use std::sync::Arc;
 
+use vortex_buffer::OwnedBuffer;
 use vortex_dtype::DType;
 use vortex_error::{vortex_err, VortexError, VortexResult};
 
-use crate::buffer::OwnedBuffer;
 use crate::stats::StatsSet;
 use crate::{Array, ArrayData, ArrayDef, AsArray, IntoArray, ToArray, TryDeserializeArrayMetadata};
 

diff --git a/vortex-array/src/view.rs b/vortex-array/src/view.rs
@@ -1,10 +1,10 @@
 use std::fmt::{Debug, Formatter};
 
 use itertools::Itertools;
+use vortex_buffer::Buffer;
 use vortex_dtype::DType;
 use vortex_error::{vortex_bail, vortex_err, VortexError, VortexResult};
 
-use crate::buffer::Buffer;
 use crate::encoding::{EncodingId, EncodingRef};
 use crate::flatbuffers::array as fb;
 use crate::stats::{EmptyStatistics, Statistics};

diff --git a/vortex-array/src/visitor.rs b/vortex-array/src/visitor.rs
@@ -1,6 +1,6 @@
+use vortex_buffer::Buffer;
 use vortex_error::VortexResult;
 
-use crate::buffer::Buffer;
 use crate::validity::Validity;
 use crate::Array;
 

diff --git a/vortex-buffer/Cargo.toml b/vortex-buffer/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+name = "vortex-buffer"
+version.workspace = true
+homepage.workspace = true
+repository.workspace = true
+authors.workspace = true
+license.workspace = true
+keywords.workspace = true
+include.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+arrow-buffer = { workspace = true }
+vortex-dtype = { path = "../vortex-dtype" }
+
+[lints]
+workspace = true
diff --git a/vortex-array/src/buffer.rs → vortex-buffer/src/lib.rs b/vortex-array/src/buffer.rs → vortex-buffer/src/lib.rs
@@ -1,8 +1,6 @@
 use arrow_buffer::Buffer as ArrowBuffer;
 use vortex_dtype::{match_each_native_ptype, NativePType};
 
-use crate::ToStatic;
-
 #[derive(Debug, Clone)]
 pub enum Buffer<'a> {
     Owned(ArrowBuffer),
@@ -45,6 +43,13 @@ impl Buffer<'_> {
             }
         }
     }
+
+    pub fn to_static(&self) -> OwnedBuffer {
+        match self {
+            Buffer::Owned(d) => Buffer::Owned(d.clone()),
+            Buffer::View(_) => Buffer::Owned(self.into()),
+        }
+    }
 }
 
 impl<'a> Buffer<'a> {
@@ -61,17 +66,6 @@ impl<'a> Buffer<'a> {
     }
 }
 
-impl ToStatic for Buffer<'_> {
-    type Static = Buffer<'static>;
-
-    fn to_static(&self) -> Self::Static {
-        match self {
-            Buffer::Owned(d) => Buffer::Owned(d.clone()),
-            Buffer::View(_) => Buffer::Owned(self.into()),
-        }
-    }
-}
-
 impl From<ArrowBuffer> for OwnedBuffer {
     fn from(value: ArrowBuffer) -> Self {
         Buffer::Owned(value)

diff --git a/vortex-ipc/Cargo.toml b/vortex-ipc/Cargo.toml
@@ -19,6 +19,7 @@ itertools = { workspace = true }
 log = { workspace = true }
 nougat = "0.2.4"
 vortex-array = { path = "../vortex-array" }
+vortex-buffer = { path = "../vortex-buffer" }
 vortex-error = { path = "../vortex-error" }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }
 vortex-dtype = { path = "../vortex-dtype" }
@@ -32,8 +33,8 @@ walkdir = { workspace = true }
 criterion = { workspace = true }
 rand = { workspace = true }
 simplelog = { workspace = true }
-vortex-fastlanes = { path = "../vortex-fastlanes" }
 vortex-alp = { path = "../vortex-alp" }
+vortex-fastlanes = { path = "../vortex-fastlanes" }
 arrow = { workspace = true }
 arrow-array = { workspace = true }
 arrow-ipc = { workspace = true, features = ["lz4"] }

diff --git a/vortex-ipc/src/reader.rs b/vortex-ipc/src/reader.rs
@@ -8,7 +8,6 @@ use flatbuffers::{root, root_unchecked};
 use log::error;
 use nougat::gat;
 use vortex::array::chunked::ChunkedArray;
-use vortex::buffer::Buffer;
 use vortex::compute::scalar_subtract::subtract_scalar;
 use vortex::compute::search_sorted::{search_sorted, SearchSortedSide};
 use vortex::compute::slice::slice;
@@ -17,6 +16,7 @@ use vortex::stats::{ArrayStatistics, Stat};
 use vortex::{
     Array, ArrayDType, ArrayView, Context, IntoArray, OwnedArray, ToArray, ToStatic, ViewContext,
 };
+use vortex_buffer::Buffer;
 use vortex_dtype::{match_each_integer_ptype, DType};
 use vortex_error::{vortex_bail, vortex_err, VortexError, VortexResult};
 use vortex_flatbuffers::ReadFlatBuffer;

diff --git a/vortex-roaring/Cargo.toml b/vortex-roaring/Cargo.toml
@@ -14,6 +14,7 @@ rust-version = { workspace = true }
 [dependencies]
 arrow-buffer = { workspace = true }
 vortex-array = { path = "../vortex-array" }
+vortex-buffer = { path = "../vortex-buffer" }
 vortex-error = { path = "../vortex-error" }
 vortex-dtype = { path = "../vortex-dtype" }
 vortex-scalar = { path = "../vortex-scalar" }

diff --git a/vortex-roaring/src/boolean/mod.rs b/vortex-roaring/src/boolean/mod.rs
@@ -4,11 +4,11 @@ use compress::roaring_encode;
 use croaring::{Bitmap, Portable};
 use serde::{Deserialize, Serialize};
 use vortex::array::bool::{Bool, BoolArray};
-use vortex::buffer::Buffer;
 use vortex::stats::ArrayStatisticsCompute;
 use vortex::validity::{ArrayValidity, LogicalValidity, Validity};
 use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor};
 use vortex::{impl_encoding, ArrayDType, ArrayFlatten, OwnedArray};
+use vortex_buffer::Buffer;
 use vortex_dtype::Nullability;
 use vortex_dtype::Nullability::NonNullable;
 use vortex_error::{vortex_bail, vortex_err};

diff --git a/vortex-roaring/src/integer/mod.rs b/vortex-roaring/src/integer/mod.rs
@@ -2,11 +2,11 @@ use compress::roaring_encode;
 use croaring::{Bitmap, Portable};
 use serde::{Deserialize, Serialize};
 use vortex::array::primitive::{Primitive, PrimitiveArray};
-use vortex::buffer::Buffer;
 use vortex::stats::ArrayStatisticsCompute;
 use vortex::validity::{ArrayValidity, LogicalValidity};
 use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor};
 use vortex::{impl_encoding, ArrayFlatten, OwnedArray};
+use vortex_buffer::Buffer;
 use vortex_dtype::Nullability::NonNullable;
 use vortex_dtype::PType;
 use vortex_error::{vortex_bail, vortex_err};