From 2526a3e631078981c0ceb168686e57dedd6a3fee Mon Sep 17 00:00:00 2001 From: lixiang <447399170@qq.com> Date: Wed, 26 Jan 2022 17:01:49 +0800 Subject: [PATCH] KE-34191 replace partition Table path (#388) * KE-34191 replace partition Table path * change pom version --- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10-token-provider/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- .../kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- .../org/apache/spark/sql/hive/TableReader.scala | 17 +++++++++++++++-- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 37 files changed, 51 insertions(+), 38 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 2994eec6f7ae9..f7c6d0bc8b957 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index f86f5a5e47d4d..94c038d9a16bd 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index d1310ef328a9a..880aa0166e13b 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index eeaabb9f3f8b4..ecec48cfecc6d 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 8f8a76801416b..a6b95a53a0e28 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 4c88784ba0c0c..176fe5acc7036 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 8cebccb9d6758..84d0d12b39c54 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index a1e1a8b374999..de45675338339 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 41a49431b14e6..9c7b3679223a2 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index 3b060b33b59bb..30b2bb4d4b9e1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index 26e142f08cc21..329c9e5c3704f 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index b6991ec7439b8..5d74815dac84c 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index 0de007709efd9..31e2260295470 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index d4bab8b4e78bd..e959c52507e10 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml index a015c01a3b963..f66ef16a9a146 100644 --- a/external/kafka-0-10-token-provider/pom.xml +++ b/external/kafka-0-10-token-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 0c26677caded9..a51097ec1517d 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 59f369e2a4082..bec195959ae8e 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 8dff4b95f6f53..40fc07976c42e 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 3abb2c9e673d4..ba72890f46166 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index a01cefb81b5cf..c930a70f26437 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index a7911f73baa4e..3c85b1e0a84f2 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 4d8952beb66d4..6054d0e8f8c10 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 594b7b7268641..32173e6627de1 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 6a898bfe41831..64d1438160e5d 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/pom.xml b/pom.xml index df374899f988b..42b854ba53726 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 2461d01fe249c..108a76ce876b7 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index f31ee54f24472..6122786270bc0 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 2b0433a6b9f57..0ce3eca3e7cc4 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 6cd8b4a2d566b..824dc83d3f2a4 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index b32bb8c263170..f19a4b8a8638c 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index b944b6e66c489..6a3919969ee75 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 11ea5e64419b9..aebb65aa8a477 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index c0435f6d7cf3a..54b84e8b56e42 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index d77a814107899..3c722a991701b 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../../pom.xml diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala index e751d7c773a05..a6b6f816cadec 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala @@ -203,7 +203,10 @@ class HadoopTableReader( path.toString + tails } - val partPath = partition.getDataLocation + var partPath = partition.getDataLocation + if (conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION) != null) { + partPath = replaceLocationWithSpecialPrefix(partPath) + } val partNum = Utilities.getPartitionDesc(partition).getPartSpec.size() val pathPatternStr = getPathPatternByPath(partNum, partPath) if (!pathPatternSet.contains(pathPatternStr)) { @@ -215,10 +218,20 @@ class HadoopTableReader( } } + def replaceLocationWithSpecialPrefix(path: Path): Path = { + val specificLocation = conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION) + val tmpPath = path.toUri.toString.replaceAll("hdfs://hacluster", specificLocation) + val newPath = new Path(tmpPath) + newPath + } + val hivePartitionRDDs = verifyPartitionPath(partitionToDeserializer) .map { case (partition, partDeserializer) => val partDesc = Utilities.getPartitionDescFromTableDesc(tableDesc, partition, true) - val partPath = partition.getDataLocation + var partPath = partition.getDataLocation + if (conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION) != null) { + partPath = replaceLocationWithSpecialPrefix(partPath) + } val inputPathStr = applyFilterIfNeeded(partPath, filterOpt) val skipHeaderLineCount = tableDesc.getProperties.getProperty("skip.header.line.count", "0").toInt diff --git a/streaming/pom.xml b/streaming/pom.xml index 81bb0fe2872ea..739d3905a8888 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 392663c1763ad..3ece75ff63b45 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.1.1-kylin-4.x-r47 + 3.1.1-kylin-4.x-r48 ../pom.xml