Update arrow 47.0.0 in DataFusion (#7587)

* Update arrow 47.0.0 * Downgrade prost substrait * Fix deprecations * Prost deprecations * Update pbjson-build * Format * Remove CardinalityAwareRowConverter * Further fixes * Ignore spill tests * Fix tests * Fix Clippy * Update pin * Review feedback * Clippy
apache · Sep 25, 2023 · 5f38135 · 5f38135
1 parent 19174e7
commit 5f38135
Show file tree

Hide file tree

Showing 30 changed files with 1,336 additions and 1,747 deletions.
diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml
@@ -47,7 +47,6 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Run doctests
         run: cargo test --doc --features avro,json
       - name: Verify Working Directory Clean

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -65,7 +65,6 @@ jobs:
       - name: Check workspace in debug mode
         run: cargo check
 
-      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Check workspace with all features
         run: cargo check --workspace --benches --features avro,json
       - name: Check Cargo.lock for datafusion-cli
@@ -96,7 +95,7 @@ jobs:
         with:
           rust-version: stable
       - name: Run tests (excluding doctests)
-        run: cargo test --lib --tests --bins --features avro,json,dictionary_expressions,backtrace
+        run: cargo test --lib --tests --bins --features avro,json,backtrace
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -177,7 +176,6 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      # Note: this does not include dictionary_expressions to reduce codegen
       - name: Run doctests
         run: |
           cargo test --doc --features avro,json
@@ -302,7 +300,7 @@ jobs:
         shell: bash
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --lib --tests --bins --features avro,json,dictionary_expressions,backtrace
+          cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
         env:
@@ -338,7 +336,7 @@ jobs:
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
-          cargo test --lib --tests --bins --features avro,json,dictionary_expressions,backtrace
+          cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
         env:

diff --git a/Cargo.toml b/Cargo.toml
@@ -47,18 +47,15 @@ rust-version = "1.70"
 version = "31.0.0"
 
 [workspace.dependencies]
-arrow = { version = "46.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
-arrow-array = { version = "46.0.0", default-features = false, features = [
-    "chrono-tz",
-] }
-arrow-buffer = { version = "46.0.0", default-features = false }
-arrow-flight = { version = "46.0.0", features = ["flight-sql-experimental"] }
-arrow-schema = { version = "46.0.0", default-features = false }
-parquet = { version = "46.0.0", features = ["arrow", "async", "object_store"] }
+arrow = { version = "47.0.0", features = ["prettyprint"] }
+arrow-array = { version = "47.0.0", default-features = false, features = ["chrono-tz"] }
+arrow-buffer = { version = "47.0.0", default-features = false }
+arrow-flight = { version = "47.0.0", features = ["flight-sql-experimental"] }
+arrow-schema = { version = "47.0.0", default-features = false }
+parquet = { version = "47.0.0", features = ["arrow", "async", "object_store"] }
 sqlparser = { version = "0.38.0", features = ["visitor"] }
 chrono = { version = "0.4.31", default-features = false }
 
-
 [profile.release]
 codegen-units = 1
 lto = true

diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.70"
 readme = "README.md"
 
 [dependencies]
-arrow = "46.0.0"
+arrow = "47.0.0"
 async-trait = "0.1.41"
 aws-config = "0.55"
 aws-credential-types = "0.55"

diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
@@ -47,12 +47,12 @@ log = "0.4"
 mimalloc = { version = "0.1", default-features = false }
 num_cpus = "1.13.0"
 object_store = { version = "0.7.0", features = ["aws"] }
-prost = { version = "0.11", default-features = false }
+prost = { version = "0.12", default-features = false }
 prost-derive = { version = "0.11", default-features = false }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.82"
 tempfile = "3"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
-tonic = "0.9"
+tonic = "0.10"
 url = "2.2"
 uuid = "1.2"
diff --git a/datafusion-examples/examples/flight_sql_server.rs b/datafusion-examples/examples/flight_sql_server.rs
@@ -20,7 +20,7 @@ use arrow::record_batch::RecordBatch;
 use arrow_flight::encode::FlightDataEncoderBuilder;
 use arrow_flight::flight_descriptor::DescriptorType;
 use arrow_flight::flight_service_server::{FlightService, FlightServiceServer};
-use arrow_flight::sql::server::FlightSqlService;
+use arrow_flight::sql::server::{FlightSqlService, PeekableFlightDataStream};
 use arrow_flight::sql::{
     ActionBeginSavepointRequest, ActionBeginSavepointResult,
     ActionBeginTransactionRequest, ActionBeginTransactionResult,
@@ -36,7 +36,7 @@ use arrow_flight::sql::{
     TicketStatementQuery,
 };
 use arrow_flight::{
-    Action, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest,
+    Action, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest,
     HandshakeResponse, IpcMessage, SchemaAsIpc, Ticket,
 };
 use arrow_schema::Schema;
@@ -547,7 +547,7 @@ impl FlightSqlService for FlightSqlServiceImpl {
     async fn do_put_statement_update(
         &self,
         _ticket: CommandStatementUpdate,
-        _request: Request<Streaming<FlightData>>,
+        _request: Request<PeekableFlightDataStream>,
     ) -> Result<i64, Status> {
         info!("do_put_statement_update");
         Err(Status::unimplemented("Implement do_put_statement_update"))
@@ -556,7 +556,7 @@ impl FlightSqlService for FlightSqlServiceImpl {
     async fn do_put_prepared_statement_query(
         &self,
         _query: CommandPreparedStatementQuery,
-        _request: Request<Streaming<FlightData>>,
+        _request: Request<PeekableFlightDataStream>,
     ) -> Result<Response<<Self as FlightService>::DoPutStream>, Status> {
         info!("do_put_prepared_statement_query");
         Err(Status::unimplemented(
@@ -567,7 +567,7 @@ impl FlightSqlService for FlightSqlServiceImpl {
     async fn do_put_prepared_statement_update(
         &self,
         _handle: CommandPreparedStatementUpdate,
-        _request: Request<Streaming<FlightData>>,
+        _request: Request<PeekableFlightDataStream>,
     ) -> Result<i64, Status> {
         info!("do_put_prepared_statement_update");
         // statements like "CREATE TABLE.." or "SET datafusion.nnn.." call this function
@@ -578,7 +578,7 @@ impl FlightSqlService for FlightSqlServiceImpl {
     async fn do_put_substrait_plan(
         &self,
         _query: CommandStatementSubstraitPlan,
-        _request: Request<Streaming<FlightData>>,
+        _request: Request<PeekableFlightDataStream>,
     ) -> Result<i64, Status> {
         info!("do_put_prepared_statement_update");
         Err(Status::unimplemented(

diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -40,9 +40,6 @@ backtrace = ["datafusion-common/backtrace"]
 compression = ["xz2", "bzip2", "flate2", "zstd", "async-compression"]
 crypto_expressions = ["datafusion-physical-expr/crypto_expressions", "datafusion-optimizer/crypto_expressions"]
 default = ["crypto_expressions", "encoding__expressions", "regex_expressions", "unicode_expressions", "compression"]
-# Enables support for non-scalar, binary operations on dictionaries
-# Note: this results in significant additional codegen
-dictionary_expressions = ["datafusion-physical-expr/dictionary_expressions", "datafusion-optimizer/dictionary_expressions"]
 encoding__expressions = ["datafusion-physical-expr/encoding_expressions"]
 # Used for testing ONLY: causes all values to hash to the same value (test for collisions)
 force_hash_collisions = []

diff --git a/datafusion/core/src/datasource/physical_plan/file_stream.rs b/datafusion/core/src/datasource/physical_plan/file_stream.rs
@@ -560,7 +560,7 @@ mod tests {
             if self.error_opening_idx.contains(&idx) {
                 Ok(futures::future::ready(internal_err!("error opening")).boxed())
             } else if self.error_scanning_idx.contains(&idx) {
-                let error = futures::future::ready(Err(ArrowError::IoError(
+                let error = futures::future::ready(Err(ArrowError::IpcError(
                     "error scanning".to_owned(),
                 )));
                 let stream = futures::stream::once(error).boxed();