From 7b0549a7620927200db0bda5bf930ffe18dbfe72 Mon Sep 17 00:00:00 2001 From: "chenliang.lu" <31469905+yabola@users.noreply.github.com> Date: Wed, 8 Jul 2020 16:24:20 +0800 Subject: [PATCH] KE-15997 close checkFile in s3 (#145) Co-authored-by: chenliang.lu --- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- .../scala/org/apache/spark/scheduler/EventLoggingListener.scala | 2 +- examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kafka-0-8-assembly/pom.xml | 2 +- external/kafka-0-8/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 41 files changed, 41 insertions(+), 41 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 61ff19d0a4b8c..71333f734d9a1 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index 0654739f5d78e..292da55179322 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 13332846eb664..22ad07bf581c7 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 5550fc611f86b..5f7784d21e236 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 28671f5dacb38..fa0293207f8a4 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 3855e4badd4c3..c5341af740707 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 6f55802c39f1a..cb9a1cb8600f6 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index c4c9d634ef491..8afac700b4e78 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 198743a5b1344..508d393ebc4e8 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala index 24b67acd0ba4e..b82e57f4efa6a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala @@ -162,7 +162,7 @@ private[spark] class EventLoggingListener( def rollEventLogFile(logRollUp: SparkListenerLogRollUp): Unit ={ closeWriter() if (logRollUp != null && logRollUp.checkTime != null) { - SparkHadoopUtil.createFile(fileSystem, new Path(new Path(logBaseDir), logRollUp.checkTime), sparkConf.get(EVENT_LOG_ALLOW_EC)) + SparkHadoopUtil.createFile(fileSystem, new Path(new Path(logBaseDir), logRollUp.checkTime), sparkConf.get(EVENT_LOG_ALLOW_EC)).close() } rollIndex += 1 val now = System.currentTimeMillis() diff --git a/examples/pom.xml b/examples/pom.xml index 9bfda7c27aeab..143dbcb8d4678 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index a05e627b38c38..23243ffa61ddf 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index fb8e174ab9ee4..8ced663b2b4d1 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index aef35c66cbbb8..1a0e6e308d09a 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 57d72f3d72b7a..bb62e264671c0 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index ea4165424d31c..d544ab0c21bb8 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index f8df02b543171..05b2872d36c7b 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 6821a4f48b235..6ec3f14e2eefa 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index e1714ac215e4f..5c586aa0fa825 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml index 865212be863d6..9684ab24eeffc 100644 --- a/external/kafka-0-8-assembly/pom.xml +++ b/external/kafka-0-8-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index 85ed19b532eff..51f1c62f418eb 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index eb5eade559441..fcd63bcd5f38b 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 738723aec7805..c4a2aafe5b001 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index fd821b00caddf..87c6eda24bc39 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index fae940f159973..efc8619203ee1 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 50bf19a2b2ad3..466f2c3023da7 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 9c05245a228e8..e47835714f837 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index b27f4be2e68ea..2e8ada927ef36 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index f3b39b3970d88..fc799ea4bfd89 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/pom.xml b/pom.xml index 5b458f38c8f16..5be677db7333f 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index bf80221e90307..4f3fee556984b 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 3a0c217a723f6..212ceaf093ce0 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index c8b7b47b12361..924008dc1e6ca 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 147c111a337f6..2b1d428a1c6a4 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index a03506a532512..24780b454569a 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index d0a302490ac73..1154c583f0eee 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index ce985abc33e38..6e95bc300f86b 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index f4bafb7781a4b..d9102cb846fda 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 723124097504c..a4aa7f9601a92 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 7d6c724d57ca6..d99e91d0fe648 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 0918e66a225c6..166b0ad3006e6 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r38 + 2.4.1-kylin-r39 ../pom.xml