diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index b7755b3d9815..3f1a7a3691e3 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.59" readme = "README.md" [dependencies] -arrow = { version = "17.0.0" } +arrow = { version = "18.0.0" } clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "9.0.0" } dirs = "4.0.0" diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 44c40d45556d..91ff44fe89da 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,7 +34,7 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -arrow-flight = { version = "17.0.0" } +arrow-flight = { version = "18.0.0" } async-trait = "0.1.41" datafusion = { path = "../datafusion/core" } futures = "0.3" diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 03809bcf52bf..318281611dfc 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -38,11 +38,11 @@ jit = ["cranelift-module"] pyarrow = ["pyo3"] [dependencies] -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } avro-rs = { version = "0.13", features = ["snappy"], optional = true } cranelift-module = { version = "0.85.0", optional = true } object_store = { version = "0.3", optional = true } ordered-float = "3.0" -parquet = { version = "17.0.0", features = ["arrow"], optional = true } +parquet = { version = "18.0.0", features = ["arrow"], optional = true } pyo3 = { version = "0.16", optional = true } sqlparser = "0.18" diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index c6cf32bd74ef..fb5e5b3f3448 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } async-trait = "0.1.41" avro-rs = { version = "0.13", features = ["snappy"], optional = true } bytes = "1.1" @@ -78,7 +78,7 @@ num_cpus = "1.13.0" object_store = "0.3.0" ordered-float = "3.0" parking_lot = "0.12" -parquet = { version = "17.0.0", features = ["arrow", "async"] } +parquet = { version = "18.0.0", features = ["arrow", "async"] } paste = "^1.0" pin-project-lite = "^0.2.7" pyo3 = { version = "0.16", optional = true } diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml index 1cc8a60e7d44..8544d88f1a0f 100644 --- a/datafusion/core/fuzz-utils/Cargo.toml +++ b/datafusion/core/fuzz-utils/Cargo.toml @@ -23,6 +23,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } env_logger = "0.9.0" rand = "0.8" diff --git a/datafusion/core/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs index 7d7f1d4f8234..6db504034524 100644 --- a/datafusion/core/src/physical_plan/hash_join.rs +++ b/datafusion/core/src/physical_plan/hash_join.rs @@ -22,11 +22,11 @@ use ahash::RandomState; use arrow::{ array::{ - as_dictionary_array, as_string_array, ArrayData, ArrayRef, BooleanArray, - Date32Array, Date64Array, DecimalArray, DictionaryArray, LargeStringArray, - PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, - TimestampSecondArray, UInt32BufferBuilder, UInt32Builder, UInt64BufferBuilder, - UInt64Builder, + as_dictionary_array, as_string_array, ArrayData, ArrayRef, BasicDecimalArray, + BooleanArray, Date32Array, Date64Array, DecimalArray, DictionaryArray, + LargeStringArray, PrimitiveArray, TimestampMicrosecondArray, + TimestampMillisecondArray, TimestampSecondArray, UInt32BufferBuilder, + UInt32Builder, UInt64BufferBuilder, UInt64Builder, }, compute, datatypes::{ diff --git a/datafusion/core/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs index 79fdcf99c2d9..81bf5c1f3d2f 100644 --- a/datafusion/core/src/physical_plan/hash_utils.rs +++ b/datafusion/core/src/physical_plan/hash_utils.rs @@ -20,9 +20,9 @@ use crate::error::{DataFusionError, Result}; use ahash::{CallHasher, RandomState}; use arrow::array::{ - Array, ArrayRef, BooleanArray, Date32Array, Date64Array, DecimalArray, - DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, - Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray, + Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array, + DecimalArray, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, + Int64Array, Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array, }; diff --git a/datafusion/core/src/physical_plan/join_utils.rs b/datafusion/core/src/physical_plan/join_utils.rs index 1c0440db80db..cbabf548c12b 100644 --- a/datafusion/core/src/physical_plan/join_utils.rs +++ b/datafusion/core/src/physical_plan/join_utils.rs @@ -170,8 +170,7 @@ fn output_join_field(old_field: &Field, join_type: &JoinType, is_left: bool) -> }; if force_nullable { - // Could cleanup after https://github.com/apache/arrow-rs/issues/1934 - Field::new(old_field.name(), old_field.data_type().clone(), true) + old_field.clone().with_nullable(true) } else { old_field.clone() } diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 4f8f1e01aa70..e0d1e0827067 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -36,6 +36,6 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "9.0.0" } sqlparser = "0.18" diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml index 8ffdf266c7fa..8cff5d646155 100644 --- a/datafusion/jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" jit = [] [dependencies] -arrow = { version = "17.0.0" } +arrow = { version = "18.0.0" } cranelift = "0.85.0" cranelift-jit = "0.85.0" cranelift-module = "0.85.0" diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 1104cbe8114b..0efd7981cc37 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } async-trait = "0.1.41" chrono = { version = "0.4", default-features = false } datafusion-common = { path = "../common", version = "9.0.0" } diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index 41a16f21c2ae..0c7bffdb7caa 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"] [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false } diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs b/datafusion/physical-expr/src/aggregate/min_max.rs index 37df7713e01d..811aa473c171 100644 --- a/datafusion/physical-expr/src/aggregate/min_max.rs +++ b/datafusion/physical-expr/src/aggregate/min_max.rs @@ -26,10 +26,11 @@ use arrow::compute; use arrow::datatypes::{DataType, TimeUnit}; use arrow::{ array::{ - ArrayRef, Date32Array, Date64Array, Float32Array, Float64Array, Int16Array, - Int32Array, Int64Array, Int8Array, LargeStringArray, StringArray, - TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, - TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array, + ArrayRef, BasicDecimalArray, Date32Array, Date64Array, Float32Array, + Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray, + StringArray, TimestampMicrosecondArray, TimestampMillisecondArray, + TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array, + UInt64Array, UInt8Array, }, datatypes::Field, }; diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs index 056af7747233..ee8b224fed79 100644 --- a/datafusion/physical-expr/src/aggregate/sum.rs +++ b/datafusion/physical-expr/src/aggregate/sum.rs @@ -26,8 +26,8 @@ use arrow::compute; use arrow::datatypes::DataType; use arrow::{ array::{ - ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, - Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array, + ArrayRef, BasicDecimalArray, Float32Array, Float64Array, Int16Array, Int32Array, + Int64Array, Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array, }, datatypes::Field, }; diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 0108f7feef92..de1909676e34 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -161,9 +161,9 @@ mod tests { use crate::expressions::col; use arrow::{ array::{ - Array, DecimalArray, Float32Array, Float64Array, Int16Array, Int32Array, - Int64Array, Int8Array, StringArray, Time64NanosecondArray, - TimestampNanosecondArray, UInt32Array, + Array, BasicDecimalArray, DecimalArray, Float32Array, Float64Array, + Int16Array, Int32Array, Int64Array, Int8Array, StringArray, + Time64NanosecondArray, TimestampNanosecondArray, UInt32Array, }, datatypes::*, util::decimal::{BasicDecimal, Decimal128}, diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs index c5178b4db7e5..2509c1d6b131 100644 --- a/datafusion/physical-expr/src/expressions/try_cast.rs +++ b/datafusion/physical-expr/src/expressions/try_cast.rs @@ -119,7 +119,8 @@ mod tests { use super::*; use crate::expressions::col; use arrow::array::{ - DecimalArray, DecimalBuilder, StringArray, Time64NanosecondArray, + BasicDecimalArray, DecimalArray, DecimalBuilder, StringArray, + Time64NanosecondArray, }; use arrow::util::decimal::{BasicDecimal, Decimal128}; use arrow::{ diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index d4ee90101777..ff6da23bdba5 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -35,7 +35,7 @@ path = "src/lib.rs" [features] [dependencies] -arrow = { version = "17.0.0" } +arrow = { version = "18.0.0" } datafusion = { path = "../core", version = "9.0.0" } datafusion-common = { path = "../common", version = "9.0.0" } datafusion-expr = { path = "../expr", version = "9.0.0" } diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml index 0dd07a5e2ee6..b213f7870537 100644 --- a/datafusion/row/Cargo.toml +++ b/datafusion/row/Cargo.toml @@ -37,7 +37,7 @@ path = "src/lib.rs" jit = ["datafusion-jit"] [dependencies] -arrow = { version = "17.0.0" } +arrow = { version = "18.0.0" } datafusion-common = { path = "../common", version = "9.0.0" } datafusion-jit = { path = "../jit", version = "9.0.0", optional = true } paste = "^1.0" diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index fa477a0b759f..4258046e723e 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -38,7 +38,7 @@ unicode_expressions = [] [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "17.0.0", features = ["prettyprint"] } +arrow = { version = "18.0.0", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "9.0.0" } datafusion-expr = { path = "../expr", version = "9.0.0" } hashbrown = "0.12"