Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Re-export Arrow and Parquet crates from DataFusion #39

Merged
merged 3 commits into from
Apr 24, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions benchmarks/Cargo.toml
Original file line number Diff line number Diff line change
@@ -31,8 +31,6 @@ simd = ["datafusion/simd"]
snmalloc = ["snmalloc-rs"]

[dependencies]
arrow = { git = "https://github.com/apache/arrow-rs", rev = "c3fe3bab9905739fdda75301dab07a18c91731bd" }
parquet = { git = "https://github.com/apache/arrow-rs", rev = "c3fe3bab9905739fdda75301dab07a18c91731bd" }
datafusion = { path = "../datafusion" }
ballista = { path = "../ballista/rust/client" }
structopt = { version = "0.3", default-features = false }
5 changes: 3 additions & 2 deletions benchmarks/src/bin/nyctaxi.rs
Original file line number Diff line number Diff line change
@@ -22,8 +22,9 @@ use std::path::PathBuf;
use std::process;
use std::time::Instant;

use arrow::datatypes::{DataType, Field, Schema};
use arrow::util::pretty;
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::util::pretty;

use datafusion::error::Result;
use datafusion::execution::context::{ExecutionConfig, ExecutionContext};

25 changes: 13 additions & 12 deletions benchmarks/src/bin/tpch.rs
Original file line number Diff line number Diff line change
@@ -28,17 +28,21 @@ use std::{

use futures::StreamExt;

use arrow::datatypes::{DataType, Field, Schema};
use arrow::util::pretty;
use ballista::context::BallistaContext;

use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::arrow::util::pretty;

use datafusion::datasource::parquet::ParquetTable;
use datafusion::datasource::{CsvFile, MemTable, TableProvider};
use datafusion::error::{DataFusionError, Result};
use datafusion::logical_plan::LogicalPlan;
use datafusion::physical_plan::collect;
use datafusion::prelude::*;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;

use datafusion::parquet::basic::Compression;
use datafusion::parquet::file::properties::WriterProperties;
use structopt::StructOpt;

#[cfg(feature = "snmalloc")]
@@ -149,9 +153,7 @@ async fn main() -> Result<()> {
}
}

async fn benchmark_datafusion(
opt: BenchmarkOpt,
) -> Result<Vec<arrow::record_batch::RecordBatch>> {
async fn benchmark_datafusion(opt: BenchmarkOpt) -> Result<Vec<RecordBatch>> {
println!("Running benchmarks with the following options: {:?}", opt);
let config = ExecutionConfig::new()
.with_concurrency(opt.concurrency)
@@ -186,7 +188,7 @@ async fn benchmark_datafusion(

let mut millis = vec![];
// run benchmark
let mut result: Vec<arrow::record_batch::RecordBatch> = Vec::with_capacity(1);
let mut result: Vec<RecordBatch> = Vec::with_capacity(1);
for i in 0..opt.iterations {
let start = Instant::now();
let plan = create_logical_plan(&mut ctx, opt.query)?;
@@ -299,7 +301,7 @@ async fn execute_query(
ctx: &mut ExecutionContext,
plan: &LogicalPlan,
debug: bool,
) -> Result<Vec<arrow::record_batch::RecordBatch>> {
) -> Result<Vec<RecordBatch>> {
if debug {
println!("Logical plan:\n{:?}", plan);
}
@@ -523,9 +525,8 @@ mod tests {
use std::env;
use std::sync::Arc;

use arrow::array::*;
use arrow::record_batch::RecordBatch;
use arrow::util::display::array_value_to_string;
use datafusion::arrow::array::*;
use datafusion::arrow::util::display::array_value_to_string;

use datafusion::logical_plan::Expr;
use datafusion::logical_plan::Expr::Cast;
1 change: 0 additions & 1 deletion datafusion-examples/Cargo.toml
Original file line number Diff line number Diff line change
@@ -29,7 +29,6 @@ publish = false


[dev-dependencies]
arrow = { git = "https://github.com/apache/arrow-rs", rev = "c3fe3bab9905739fdda75301dab07a18c91731bd" }
arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "c3fe3bab9905739fdda75301dab07a18c91731bd" }
datafusion = { path = "../datafusion" }
prost = "0.7"
4 changes: 2 additions & 2 deletions datafusion-examples/examples/csv_sql.rs
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use arrow::util::pretty;
use datafusion::arrow::util::pretty;

use datafusion::error::Result;
use datafusion::prelude::*;
@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
// create local execution context
let mut ctx = ExecutionContext::new();

let testdata = arrow::util::test_util::arrow_test_data();
let testdata = datafusion::arrow::util::test_util::arrow_test_data();

// register csv file with the execution context
ctx.register_csv(
4 changes: 2 additions & 2 deletions datafusion-examples/examples/dataframe.rs
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use arrow::util::pretty;
use datafusion::arrow::util::pretty;

use datafusion::error::Result;
use datafusion::prelude::*;
@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
// create local execution context
let mut ctx = ExecutionContext::new();

let testdata = arrow::util::test_util::parquet_test_data();
let testdata = datafusion::arrow::util::test_util::parquet_test_data();

let filename = &format!("{}/alltypes_plain.parquet", testdata);

8 changes: 4 additions & 4 deletions datafusion-examples/examples/dataframe_in_memory.rs
Original file line number Diff line number Diff line change
@@ -17,10 +17,10 @@

use std::sync::Arc;

use arrow::array::{Int32Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::RecordBatch;
use arrow::util::pretty;
use datafusion::arrow::array::{Int32Array, StringArray};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::arrow::util::pretty;

use datafusion::datasource::MemTable;
use datafusion::error::Result;
6 changes: 3 additions & 3 deletions datafusion-examples/examples/flight_client.rs
Original file line number Diff line number Diff line change
@@ -18,8 +18,8 @@
use std::convert::TryFrom;
use std::sync::Arc;

use arrow::datatypes::Schema;
use arrow::util::pretty;
use datafusion::arrow::datatypes::Schema;
use datafusion::arrow::util::pretty;

use arrow_flight::flight_descriptor;
use arrow_flight::flight_service_client::FlightServiceClient;
@@ -31,7 +31,7 @@ use arrow_flight::{FlightDescriptor, Ticket};
/// This example is run along-side the example `flight_server`.
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let testdata = arrow::util::test_util::parquet_test_data();
let testdata = datafusion::arrow::util::test_util::parquet_test_data();

// Create Flight client
let mut client = FlightServiceClient::connect("http://localhost:50051").await?;
6 changes: 3 additions & 3 deletions datafusion-examples/examples/flight_server.rs
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ impl FlightService for FlightServiceImpl {

let table = ParquetTable::try_new(&request.path[0], num_cpus::get()).unwrap();

let options = arrow::ipc::writer::IpcWriteOptions::default();
let options = datafusion::arrow::ipc::writer::IpcWriteOptions::default();
let schema_result = arrow_flight::utils::flight_schema_from_arrow_schema(
table.schema().as_ref(),
&options,
@@ -87,7 +87,7 @@ impl FlightService for FlightServiceImpl {
// create local execution context
let mut ctx = ExecutionContext::new();

let testdata = arrow::util::test_util::parquet_test_data();
let testdata = datafusion::arrow::util::test_util::parquet_test_data();

// register parquet file with the execution context
ctx.register_parquet(
@@ -106,7 +106,7 @@ impl FlightService for FlightServiceImpl {
}

// add an initial FlightData message that sends schema
let options = arrow::ipc::writer::IpcWriteOptions::default();
let options = datafusion::arrow::ipc::writer::IpcWriteOptions::default();
let schema_flight_data =
arrow_flight::utils::flight_data_from_arrow_schema(
&df.schema().clone().into(),
4 changes: 2 additions & 2 deletions datafusion-examples/examples/parquet_sql.rs
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use arrow::util::pretty;
use datafusion::arrow::util::pretty;

use datafusion::error::Result;
use datafusion::prelude::*;
@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
// create local execution context
let mut ctx = ExecutionContext::new();

let testdata = arrow::util::test_util::parquet_test_data();
let testdata = datafusion::arrow::util::test_util::parquet_test_data();

// register parquet file with the execution context
ctx.register_parquet(
4 changes: 2 additions & 2 deletions datafusion-examples/examples/simple_udaf.rs
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@

/// In this example we will declare a single-type, single return type UDAF that computes the geometric mean.
/// The geometric mean is described here: https://en.wikipedia.org/wiki/Geometric_mean
use arrow::{
use datafusion::arrow::{
array::Float32Array, array::Float64Array, datatypes::DataType,
record_batch::RecordBatch,
};
@@ -28,7 +28,7 @@ use std::sync::Arc;

// create local execution context with an in-memory table
fn create_context() -> Result<ExecutionContext> {
use arrow::datatypes::{Field, Schema};
use datafusion::arrow::datatypes::{Field, Schema};
use datafusion::datasource::MemTable;
// define a schema.
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, false)]));
4 changes: 2 additions & 2 deletions datafusion-examples/examples/simple_udf.rs
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use arrow::{
use datafusion::arrow::{
array::{ArrayRef, Float32Array, Float64Array},
datatypes::DataType,
record_batch::RecordBatch,
@@ -28,7 +28,7 @@ use std::sync::Arc;

// create local execution context with an in-memory table
fn create_context() -> Result<ExecutionContext> {
use arrow::datatypes::{Field, Schema};
use datafusion::arrow::datatypes::{Field, Schema};
use datafusion::datasource::MemTable;
// define a schema.
let schema = Arc::new(Schema::new(vec![
5 changes: 4 additions & 1 deletion datafusion/src/lib.rs
Original file line number Diff line number Diff line change
@@ -183,7 +183,6 @@
//!
//! you can find examples of each of them in examples section.
extern crate arrow;
extern crate sqlparser;

pub mod catalog;
@@ -200,6 +199,10 @@ pub mod scalar;
pub mod sql;
pub mod variable;

// re-export dependencies from arrow-rs to minimise version maintenance for crate users
pub use arrow;
pub use parquet;

#[cfg(test)]
pub mod test;