diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 8c9e6abeec5f..4def3f39c554 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -42,7 +42,7 @@ apache-avro = { version = "0.14", features = ["snappy"], optional = true } arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "30c94dbf1c422f81f8520b9956e96ab7b53c3f47", features = ["prettyprint"], optional = false } avro-rs = { version = "0.13", features = ["snappy"], optional = true } cranelift-module = { version = "0.86.1", optional = true } -object_store = { git = "https://github.com/apache/arrow-rs.git", rev = "30c94dbf1c422f81f8520b9956e96ab7b53c3f47", features = [], optional = true } +object_store = { version = "0.3", optional = true } ordered-float = "3.0" parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "30c94dbf1c422f81f8520b9956e96ab7b53c3f47", features = ["arrow"], optional = true } pyo3 = { version = "0.16", optional = true } diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index bb9badfc368c..f1399091b687 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -75,7 +75,7 @@ lazy_static = { version = "^1.4.0" } log = "^0.4" num-traits = { version = "0.2", optional = true } num_cpus = "1.13.0" -object_store = { git = "https://github.com/apache/arrow-rs.git", rev = "30c94dbf1c422f81f8520b9956e96ab7b53c3f47", features = [], optional = false } +object_store = "0.3.0" ordered-float = "3.0" parking_lot = "0.12" parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "30c94dbf1c422f81f8520b9956e96ab7b53c3f47", features = ["arrow", "async"], optional = false } diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs index b5bae663728f..04b0b9d523df 100644 --- a/datafusion/core/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -575,8 +575,7 @@ mod tests { use futures::StreamExt; use object_store::local::LocalFileSystem; use object_store::path::Path; - use object_store::{GetResult, ListResult, MultipartId}; - use tokio::io::AsyncWrite; + use object_store::{GetResult, ListResult}; #[tokio::test] async fn read_merged_batches() -> Result<()> { @@ -650,22 +649,6 @@ mod tests { Err(object_store::Error::NotImplemented) } - async fn put_multipart( - &self, - _location: &Path, - ) -> object_store::Result<(MultipartId, Box)> - { - Err(object_store::Error::NotImplemented) - } - - async fn abort_multipart( - &self, - _location: &Path, - _multipart_id: &MultipartId, - ) -> object_store::Result<()> { - Err(object_store::Error::NotImplemented) - } - async fn get(&self, _location: &Path) -> object_store::Result { Err(object_store::Error::NotImplemented) } diff --git a/datafusion/core/src/physical_plan/file_format/chunked_store.rs b/datafusion/core/src/physical_plan/file_format/chunked_store.rs index 1a48804a2c55..216926b06713 100644 --- a/datafusion/core/src/physical_plan/file_format/chunked_store.rs +++ b/datafusion/core/src/physical_plan/file_format/chunked_store.rs @@ -20,12 +20,11 @@ use bytes::Bytes; use futures::stream::BoxStream; use futures::StreamExt; use object_store::path::Path; +use object_store::Result; use object_store::{GetResult, ListResult, ObjectMeta, ObjectStore}; -use object_store::{MultipartId, Result}; use std::fmt::{Debug, Display, Formatter}; use std::ops::Range; use std::sync::Arc; -use tokio::io::AsyncWrite; /// Wraps a [`ObjectStore`] and makes its get response return chunks /// @@ -54,21 +53,6 @@ impl ObjectStore for ChunkedStore { self.inner.put(location, bytes).await } - async fn put_multipart( - &self, - location: &Path, - ) -> Result<(MultipartId, Box)> { - self.inner.put_multipart(location).await - } - - async fn abort_multipart( - &self, - location: &Path, - multipart_id: &MultipartId, - ) -> Result<()> { - self.inner.abort_multipart(location, multipart_id).await - } - async fn get(&self, location: &Path) -> Result { let bytes = self.inner.get(location).await?.bytes().await?; let mut offset = 0; diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs index fca9b9a43b1c..821d174f2d99 100644 --- a/datafusion/core/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -40,10 +40,7 @@ use datafusion::{ use datafusion_common::ScalarValue; use futures::stream::BoxStream; use futures::{stream, StreamExt}; -use object_store::{ - path::Path, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore, -}; -use tokio::io::AsyncWrite; +use object_store::{path::Path, GetResult, ListResult, ObjectMeta, ObjectStore}; #[tokio::test] async fn parquet_distinct_partition_col() -> Result<()> { @@ -519,21 +516,6 @@ impl ObjectStore for MirroringObjectStore { unimplemented!() } - async fn put_multipart( - &self, - _location: &Path, - ) -> object_store::Result<(MultipartId, Box)> { - unimplemented!() - } - - async fn abort_multipart( - &self, - _location: &Path, - _multipart_id: &MultipartId, - ) -> object_store::Result<()> { - unimplemented!() - } - async fn get(&self, location: &Path) -> object_store::Result { self.files.iter().find(|x| *x == location.as_ref()).unwrap(); let path = std::path::PathBuf::from(&self.mirrored_file);