From 2c47760df93541e8376b373f4bf6a84dde5a41e1 Mon Sep 17 00:00:00 2001 From: weiting-chen Date: Thu, 1 Jul 2021 15:40:35 +0800 Subject: [PATCH 1/3] Fix Hadoop version issue --- arrow-data-source/pom.xml | 1 + native-sql-engine/core/pom.xml | 1 + pom.xml | 142 ++++++++++++++++++--------------- 3 files changed, 78 insertions(+), 66 deletions(-) diff --git a/arrow-data-source/pom.xml b/arrow-data-source/pom.xml index 285a85cb1..3f72871f2 100644 --- a/arrow-data-source/pom.xml +++ b/arrow-data-source/pom.xml @@ -50,6 +50,7 @@ org.apache.hadoop hadoop-client + ${hadoop.version} provided diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml index 1c8986677..e7263e99a 100644 --- a/native-sql-engine/core/pom.xml +++ b/native-sql-engine/core/pom.xml @@ -255,6 +255,7 @@ org.apache.hadoop hadoop-client + ${hadoop.version} provided diff --git a/pom.xml b/pom.xml index 9f24f543a..1c8035e8f 100644 --- a/pom.xml +++ b/pom.xml @@ -29,13 +29,88 @@ + + arrow-data-source + native-sql-engine/core + shims + + + + + spark + + true + + + ${spark311.version} + + + + spark-3.1.1 + + ${spark311.version} + + + + hadoop-2.7.4 + + true + + + 2.7.4 + + + + hadoop-3.2 + + 3.2.0 + + + + incremental-scala-compiler + + true + + + incremental + + + + full-scala-compiler + + false + + + all + + + + arrow-netty + + false + + + arrow-memory-netty + + + + arrow-unsafe + + true + + + arrow-memory-unsafe + + + + 2.12.10 2.12 3.1.1 4.0.0 arrow-memory-netty - 2.7.4 + ${hadoop.version} UTF-8 UTF-8 ${project.basedir}/script @@ -50,12 +125,6 @@ 3.1.1 - - arrow-data-source - native-sql-engine/core - shims - - @@ -189,63 +258,4 @@ - - - spark - - true - - - ${spark311.version} - - - - spark-3.1.1 - - ${spark311.version} - - - - hadoop-3.2 - - 3.2.0 - - - - incremental-scala-compiler - - true - - - incremental - - - - full-scala-compiler - - false - - - all - - - - arrow-netty - - false - - - arrow-memory-netty - - - - arrow-unsafe - - true - - - arrow-memory-unsafe - - - From 21f54a96e6a29797fb9ea4f81114031351b025cb Mon Sep 17 00:00:00 2001 From: weiting-chen Date: Fri, 2 Jul 2021 13:53:29 +0800 Subject: [PATCH 2/3] Fix NullPointer Issue. --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1c8035e8f..8ab01b467 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,9 @@ hadoop-2.7.4 - true + + !hadoop.version + 2.7.4 From 3dc9c889ea0e7d90423cb10c712d22a425f2fdf9 Mon Sep 17 00:00:00 2001 From: weiting-chen Date: Mon, 5 Jul 2021 09:36:40 +0800 Subject: [PATCH 3/3] Update mvn version check and enable arrow-memory unsafe by default --- pom.xml | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 8ab01b467..9e5aa76e1 100644 --- a/pom.xml +++ b/pom.xml @@ -92,18 +92,10 @@ false + arrow-netty arrow-memory-netty - - arrow-unsafe - - true - - - arrow-memory-unsafe - - @@ -111,7 +103,7 @@ 2.12 3.1.1 4.0.0 - arrow-memory-netty + arrow-memory-unsafe ${hadoop.version} UTF-8 UTF-8 @@ -260,4 +252,29 @@ + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.0.1 + + + enforce-versions + + enforce + + + + + 3.6.3 + + + + + + + + +