From 7090bc1f43fd5ae5e67214f84de0276ee6e5df79 Mon Sep 17 00:00:00 2001 From: sychen Date: Thu, 25 Apr 2024 16:02:01 +0800 Subject: [PATCH] [SPARK-47981][BUILD] Upgrade `Arrow` to 16.0.0 ### What changes were proposed in this pull request? The pr aims to upgrade `Arrow` from `15.0.2` to `16.0.0`. ### Why are the changes needed? https://arrow.apache.org/release/16.0.0.html SPARK-46718 and SPARK-47531 upgraded the arrow version from 14 to 15, and 15 introduced the `eclipse-collections` dependency. https://github.com/apache/arrow/issues/40896 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? GA ### Was this patch authored or co-authored using generative AI tooling? No Closes #46214 from cxzl25/SPARK-47981. Authored-by: sychen Signed-off-by: Kent Yao --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 12 ++++++------ pom.xml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index c1adff73d3399..f6adb6d18b858 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -16,10 +16,11 @@ antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar aopalliance-repackaged/3.0.3//aopalliance-repackaged-3.0.3.jar arpack/3.0.3//arpack-3.0.3.jar arpack_combined_all/0.1//arpack_combined_all-0.1.jar -arrow-format/15.0.2//arrow-format-15.0.2.jar -arrow-memory-core/15.0.2//arrow-memory-core-15.0.2.jar -arrow-memory-netty/15.0.2//arrow-memory-netty-15.0.2.jar -arrow-vector/15.0.2//arrow-vector-15.0.2.jar +arrow-format/16.0.0//arrow-format-16.0.0.jar +arrow-memory-core/16.0.0//arrow-memory-core-16.0.0.jar +arrow-memory-netty-buffer-patch/16.0.0//arrow-memory-netty-buffer-patch-16.0.0.jar +arrow-memory-netty/16.0.0//arrow-memory-netty-16.0.0.jar +arrow-vector/16.0.0//arrow-vector-16.0.0.jar audience-annotations/0.12.0//audience-annotations-0.12.0.jar avro-ipc/1.11.3//avro-ipc-1.11.3.jar avro-mapred/1.11.3//avro-mapred-1.11.3.jar @@ -33,6 +34,7 @@ breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar bundle/2.24.6//bundle-2.24.6.jar cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar +checker-qual/3.42.0//checker-qual-3.42.0.jar chill-java/0.10.0//chill-java-0.10.0.jar chill_2.13/0.10.0//chill_2.13-0.10.0.jar commons-cli/1.6.0//commons-cli-1.6.0.jar @@ -62,8 +64,6 @@ derby/10.16.1.1//derby-10.16.1.1.jar derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar derbytools/10.16.1.1//derbytools-10.16.1.1.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar -eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar -eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar esdk-obs-java/3.20.4.2//esdk-obs-java-3.20.4.2.jar flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar gcs-connector/hadoop3-2.2.21/shaded/gcs-connector-hadoop3-2.2.21-shaded.jar diff --git a/pom.xml b/pom.xml index 03c6b757ab025..338b4050e0f8d 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ ./python/pyspark/sql/pandas/utils.py, ./python/packaging/classic/setup.py and ./python/packaging/connect/setup.py too. --> - 15.0.2 + 16.0.0 3.0.0-M1