From 2526a3e631078981c0ceb168686e57dedd6a3fee Mon Sep 17 00:00:00 2001
From: lixiang <447399170@qq.com>
Date: Wed, 26 Jan 2022 17:01:49 +0800
Subject: [PATCH] KE-34191 replace partition Table path (#388)
* KE-34191 replace partition Table path
* change pom version
---
assembly/pom.xml | 2 +-
common/kvstore/pom.xml | 2 +-
common/network-common/pom.xml | 2 +-
common/network-shuffle/pom.xml | 2 +-
common/network-yarn/pom.xml | 2 +-
common/sketch/pom.xml | 2 +-
common/tags/pom.xml | 2 +-
common/unsafe/pom.xml | 2 +-
core/pom.xml | 2 +-
examples/pom.xml | 2 +-
external/avro/pom.xml | 2 +-
external/docker-integration-tests/pom.xml | 2 +-
external/kafka-0-10-assembly/pom.xml | 2 +-
external/kafka-0-10-sql/pom.xml | 2 +-
external/kafka-0-10-token-provider/pom.xml | 2 +-
external/kafka-0-10/pom.xml | 2 +-
external/kinesis-asl-assembly/pom.xml | 2 +-
external/kinesis-asl/pom.xml | 2 +-
external/spark-ganglia-lgpl/pom.xml | 2 +-
graphx/pom.xml | 2 +-
hadoop-cloud/pom.xml | 2 +-
launcher/pom.xml | 2 +-
mllib-local/pom.xml | 2 +-
mllib/pom.xml | 2 +-
pom.xml | 2 +-
repl/pom.xml | 2 +-
resource-managers/kubernetes/core/pom.xml | 2 +-
.../kubernetes/integration-tests/pom.xml | 2 +-
resource-managers/mesos/pom.xml | 2 +-
resource-managers/yarn/pom.xml | 2 +-
sql/catalyst/pom.xml | 2 +-
sql/core/pom.xml | 2 +-
sql/hive-thriftserver/pom.xml | 2 +-
sql/hive/pom.xml | 2 +-
.../org/apache/spark/sql/hive/TableReader.scala | 17 +++++++++++++++--
streaming/pom.xml | 2 +-
tools/pom.xml | 2 +-
37 files changed, 51 insertions(+), 38 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 2994eec6f7ae9..f7c6d0bc8b957 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index f86f5a5e47d4d..94c038d9a16bd 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index d1310ef328a9a..880aa0166e13b 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index eeaabb9f3f8b4..ecec48cfecc6d 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 8f8a76801416b..a6b95a53a0e28 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 4c88784ba0c0c..176fe5acc7036 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 8cebccb9d6758..84d0d12b39c54 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index a1e1a8b374999..de45675338339 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/core/pom.xml b/core/pom.xml
index 41a49431b14e6..9c7b3679223a2 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/examples/pom.xml b/examples/pom.xml
index 3b060b33b59bb..30b2bb4d4b9e1 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 26e142f08cc21..329c9e5c3704f 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index b6991ec7439b8..5d74815dac84c 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 0de007709efd9..31e2260295470 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index d4bab8b4e78bd..e959c52507e10 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index a015c01a3b963..f66ef16a9a146 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 0c26677caded9..a51097ec1517d 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 59f369e2a4082..bec195959ae8e 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 8dff4b95f6f53..40fc07976c42e 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index 3abb2c9e673d4..ba72890f46166 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/graphx/pom.xml b/graphx/pom.xml
index a01cefb81b5cf..c930a70f26437 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index a7911f73baa4e..3c85b1e0a84f2 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 4d8952beb66d4..6054d0e8f8c10 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 594b7b7268641..32173e6627de1 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 6a898bfe41831..64d1438160e5d 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/pom.xml b/pom.xml
index df374899f988b..42b854ba53726 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
pom
Spark Project Parent POM
http://spark.apache.org/
diff --git a/repl/pom.xml b/repl/pom.xml
index 2461d01fe249c..108a76ce876b7 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index f31ee54f24472..6122786270bc0 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../../pom.xml
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 2b0433a6b9f57..0ce3eca3e7cc4 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../../pom.xml
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 6cd8b4a2d566b..824dc83d3f2a4 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index b32bb8c263170..f19a4b8a8638c 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index b944b6e66c489..6a3919969ee75 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 11ea5e64419b9..aebb65aa8a477 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index c0435f6d7cf3a..54b84e8b56e42 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index d77a814107899..3c722a991701b 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../../pom.xml
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index e751d7c773a05..a6b6f816cadec 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -203,7 +203,10 @@ class HadoopTableReader(
path.toString + tails
}
- val partPath = partition.getDataLocation
+ var partPath = partition.getDataLocation
+ if (conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION) != null) {
+ partPath = replaceLocationWithSpecialPrefix(partPath)
+ }
val partNum = Utilities.getPartitionDesc(partition).getPartSpec.size()
val pathPatternStr = getPathPatternByPath(partNum, partPath)
if (!pathPatternSet.contains(pathPatternStr)) {
@@ -215,10 +218,20 @@ class HadoopTableReader(
}
}
+ def replaceLocationWithSpecialPrefix(path: Path): Path = {
+ val specificLocation = conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION)
+ val tmpPath = path.toUri.toString.replaceAll("hdfs://hacluster", specificLocation)
+ val newPath = new Path(tmpPath)
+ newPath
+ }
+
val hivePartitionRDDs = verifyPartitionPath(partitionToDeserializer)
.map { case (partition, partDeserializer) =>
val partDesc = Utilities.getPartitionDescFromTableDesc(tableDesc, partition, true)
- val partPath = partition.getDataLocation
+ var partPath = partition.getDataLocation
+ if (conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION) != null) {
+ partPath = replaceLocationWithSpecialPrefix(partPath)
+ }
val inputPathStr = applyFilterIfNeeded(partPath, filterOpt)
val skipHeaderLineCount =
tableDesc.getProperties.getProperty("skip.header.line.count", "0").toInt
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 81bb0fe2872ea..739d3905a8888 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml
diff --git a/tools/pom.xml b/tools/pom.xml
index 392663c1763ad..3ece75ff63b45 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.12
- 3.1.1-kylin-4.x-r47
+ 3.1.1-kylin-4.x-r48
../pom.xml