From 65c0a7812b472147c615fb4fe779da9d0a11ff18 Mon Sep 17 00:00:00 2001 From: "igor.calabria" Date: Fri, 18 Oct 2019 09:18:45 -0700 Subject: [PATCH] [SPARK-27812][K8S][2.4] Bump K8S client version to 4.6.1 # What changes were proposed in this pull request? Backport of #26093 to `branch-2.4` ### Why are the changes needed? https://issues.apache.org/jira/browse/SPARK-27812 https://issues.apache.org/jira/browse/SPARK-27927 We need this fix https://github.com/fabric8io/kubernetes-client/pull/1768 that was released on version 4.6 of the client. The root cause of the problem is better explained in https://github.com/apache/spark/pull/25785 ### Does this PR introduce any user-facing change? No ### How was this patch tested? This patch was tested manually using a simple pyspark job ```python from pyspark.sql import SparkSession if __name__ == '__main__': spark = SparkSession.builder.getOrCreate() ``` The expected behaviour of this "job" is that both python's and jvm's process exit automatically after the main runs. This is the case for spark versions <= 2.4. On version 2.4.3, the jvm process hangs because there's a non daemon thread running ``` "OkHttp WebSocket https://10.96.0.1/..." #121 prio=5 os_prio=0 tid=0x00007fb27c005800 nid=0x24b waiting on condition [0x00007fb300847000] "OkHttp WebSocket https://10.96.0.1/..." #117 prio=5 os_prio=0 tid=0x00007fb28c004000 nid=0x247 waiting on condition [0x00007fb300e4b000] ``` This is caused by a bug on `kubernetes-client` library, which is fixed on the version that we are upgrading to. When the mentioned job is run with this patch applied, the behaviour from spark <= 2.4.0 is restored and both processes terminate successfully Closes #26152 from igorcalabria/k8s-client-update-2.4. Authored-by: igor.calabria Signed-off-by: Dongjoon Hyun --- dev/deps/spark-deps-hadoop-2.6 | 11 +++++------ dev/deps/spark-deps-hadoop-2.7 | 11 +++++------ dev/deps/spark-deps-hadoop-3.1 | 11 +++++------ resource-managers/kubernetes/core/pom.xml | 8 +------- .../kubernetes/integration-tests/pom.xml | 2 +- 5 files changed, 17 insertions(+), 26 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 8e46582011aa3..b4239a562900a 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -100,7 +100,6 @@ jackson-xc-1.9.13.jar janino-3.0.9.jar javassist-3.18.1-GA.jar javax.annotation-api-1.2.jar -javax.el-3.0.1-b11.jar javax.inject-1.jar javax.inject-2.4.0-b34.jar javax.servlet-api-3.1.0.jar @@ -131,9 +130,9 @@ jta-1.1.jar jtransforms-2.4.0.jar jul-to-slf4j-1.7.16.jar kryo-shaded-4.0.2.jar -kubernetes-client-4.4.2.jar -kubernetes-model-4.4.2.jar -kubernetes-model-common-4.4.2.jar +kubernetes-client-4.6.1.jar +kubernetes-model-4.6.1.jar +kubernetes-model-common-4.6.1.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar libthrift-0.9.3.jar @@ -151,8 +150,8 @@ minlog-1.3.0.jar netty-3.9.9.Final.jar netty-all-4.1.42.Final.jar objenesis-2.5.1.jar -okhttp-3.8.1.jar -okio-1.13.0.jar +okhttp-3.12.0.jar +okio-1.15.0.jar opencsv-2.3.jar orc-core-1.5.5-nohive.jar orc-mapreduce-1.5.5-nohive.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 87b7c9c74f888..47fe465ce8f73 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -100,7 +100,6 @@ jackson-xc-1.9.13.jar janino-3.0.9.jar javassist-3.18.1-GA.jar javax.annotation-api-1.2.jar -javax.el-3.0.1-b11.jar javax.inject-1.jar javax.inject-2.4.0-b34.jar javax.servlet-api-3.1.0.jar @@ -132,9 +131,9 @@ jta-1.1.jar jtransforms-2.4.0.jar jul-to-slf4j-1.7.16.jar kryo-shaded-4.0.2.jar -kubernetes-client-4.4.2.jar -kubernetes-model-4.4.2.jar -kubernetes-model-common-4.4.2.jar +kubernetes-client-4.6.1.jar +kubernetes-model-4.6.1.jar +kubernetes-model-common-4.6.1.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar libthrift-0.9.3.jar @@ -152,8 +151,8 @@ minlog-1.3.0.jar netty-3.9.9.Final.jar netty-all-4.1.42.Final.jar objenesis-2.5.1.jar -okhttp-3.8.1.jar -okio-1.13.0.jar +okhttp-3.12.0.jar +okio-1.15.0.jar opencsv-2.3.jar orc-core-1.5.5-nohive.jar orc-mapreduce-1.5.5-nohive.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 6789cb9f34e8b..4cf26f94beed1 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -100,7 +100,6 @@ jackson-module-scala_2.11-2.6.7.1.jar janino-3.0.9.jar javassist-3.18.1-GA.jar javax.annotation-api-1.2.jar -javax.el-3.0.1-b11.jar javax.inject-1.jar javax.inject-2.4.0-b34.jar javax.servlet-api-3.1.0.jar @@ -148,9 +147,9 @@ kerby-pkix-1.0.1.jar kerby-util-1.0.1.jar kerby-xdr-1.0.1.jar kryo-shaded-4.0.2.jar -kubernetes-client-4.4.2.jar -kubernetes-model-4.4.2.jar -kubernetes-model-common-4.4.2.jar +kubernetes-client-4.6.1.jar +kubernetes-model-4.6.1.jar +kubernetes-model-common-4.6.1.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar libthrift-0.9.3.jar @@ -171,8 +170,8 @@ netty-all-4.1.42.Final.jar nimbus-jose-jwt-4.41.1.jar objenesis-2.5.1.jar okhttp-2.7.5.jar -okhttp-3.8.1.jar -okio-1.13.0.jar +okhttp-3.12.0.jar +okio-1.15.0.jar opencsv-2.3.jar orc-core-1.5.5-nohive.jar orc-mapreduce-1.5.5-nohive.jar diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 852fa06a83364..56e1afc89e7c3 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -29,7 +29,7 @@ Spark Project Kubernetes kubernetes - 4.4.2 + 4.6.1 @@ -83,12 +83,6 @@ - - com.squareup.okhttp3 - okhttp - 3.8.1 - - org.mockito mockito-core diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 69d68f30d501f..f5bb1f8629124 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -29,7 +29,7 @@ 1.3.0 1.4.0 - 4.4.2 + 4.6.1 3.2.2 1.0 kubernetes-integration-tests