[SPARK-27812][K8S][2.4] Bump K8S client version to 4.6.1

# What changes were proposed in this pull request? Backport of #26093 to `branch-2.4` ### Why are the changes needed? https://issues.apache.org/jira/browse/SPARK-27812 https://issues.apache.org/jira/browse/SPARK-27927 We need this fix fabric8io/kubernetes-client#1768 that was released on version 4.6 of the client. The root cause of the problem is better explained in #25785 ### Does this PR introduce any user-facing change? No ### How was this patch tested? This patch was tested manually using a simple pyspark job ```python from pyspark.sql import SparkSession if __name__ == '__main__': spark = SparkSession.builder.getOrCreate() ``` The expected behaviour of this "job" is that both python's and jvm's process exit automatically after the main runs. This is the case for spark versions <= 2.4. On version 2.4.3, the jvm process hangs because there's a non daemon thread running ``` "OkHttp WebSocket https://10.96.0.1/..." #121 prio=5 os_prio=0 tid=0x00007fb27c005800 nid=0x24b waiting on condition [0x00007fb300847000] "OkHttp WebSocket https://10.96.0.1/..." #117 prio=5 os_prio=0 tid=0x00007fb28c004000 nid=0x247 waiting on condition [0x00007fb300e4b000] ``` This is caused by a bug on `kubernetes-client` library, which is fixed on the version that we are upgrading to. When the mentioned job is run with this patch applied, the behaviour from spark <= 2.4.0 is restored and both processes terminate successfully Closes #26152 from igorcalabria/k8s-client-update-2.4. Authored-by: igor.calabria <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
apache · Oct 18, 2019 · 65c0a78 · 65c0a78
1 parent 90139f6
commit 65c0a78
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 26 deletions.
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
@@ -100,7 +100,6 @@ jackson-xc-1.9.13.jar
 janino-3.0.9.jar
 javassist-3.18.1-GA.jar
 javax.annotation-api-1.2.jar
-javax.el-3.0.1-b11.jar
 javax.inject-1.jar
 javax.inject-2.4.0-b34.jar
 javax.servlet-api-3.1.0.jar
@@ -131,9 +130,9 @@ jta-1.1.jar
 jtransforms-2.4.0.jar
 jul-to-slf4j-1.7.16.jar
 kryo-shaded-4.0.2.jar
-kubernetes-client-4.4.2.jar
-kubernetes-model-4.4.2.jar
-kubernetes-model-common-4.4.2.jar
+kubernetes-client-4.6.1.jar
+kubernetes-model-4.6.1.jar
+kubernetes-model-common-4.6.1.jar
 leveldbjni-all-1.8.jar
 libfb303-0.9.3.jar
 libthrift-0.9.3.jar
@@ -151,8 +150,8 @@ minlog-1.3.0.jar
 netty-3.9.9.Final.jar
 netty-all-4.1.42.Final.jar
 objenesis-2.5.1.jar
-okhttp-3.8.1.jar
-okio-1.13.0.jar
+okhttp-3.12.0.jar
+okio-1.15.0.jar
 opencsv-2.3.jar
 orc-core-1.5.5-nohive.jar
 orc-mapreduce-1.5.5-nohive.jar

diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
@@ -100,7 +100,6 @@ jackson-xc-1.9.13.jar
 janino-3.0.9.jar
 javassist-3.18.1-GA.jar
 javax.annotation-api-1.2.jar
-javax.el-3.0.1-b11.jar
 javax.inject-1.jar
 javax.inject-2.4.0-b34.jar
 javax.servlet-api-3.1.0.jar
@@ -132,9 +131,9 @@ jta-1.1.jar
 jtransforms-2.4.0.jar
 jul-to-slf4j-1.7.16.jar
 kryo-shaded-4.0.2.jar
-kubernetes-client-4.4.2.jar
-kubernetes-model-4.4.2.jar
-kubernetes-model-common-4.4.2.jar
+kubernetes-client-4.6.1.jar
+kubernetes-model-4.6.1.jar
+kubernetes-model-common-4.6.1.jar
 leveldbjni-all-1.8.jar
 libfb303-0.9.3.jar
 libthrift-0.9.3.jar
@@ -152,8 +151,8 @@ minlog-1.3.0.jar
 netty-3.9.9.Final.jar
 netty-all-4.1.42.Final.jar
 objenesis-2.5.1.jar
-okhttp-3.8.1.jar
-okio-1.13.0.jar
+okhttp-3.12.0.jar
+okio-1.15.0.jar
 opencsv-2.3.jar
 orc-core-1.5.5-nohive.jar
 orc-mapreduce-1.5.5-nohive.jar

diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1
@@ -100,7 +100,6 @@ jackson-module-scala_2.11-2.6.7.1.jar
 janino-3.0.9.jar
 javassist-3.18.1-GA.jar
 javax.annotation-api-1.2.jar
-javax.el-3.0.1-b11.jar
 javax.inject-1.jar
 javax.inject-2.4.0-b34.jar
 javax.servlet-api-3.1.0.jar
@@ -148,9 +147,9 @@ kerby-pkix-1.0.1.jar
 kerby-util-1.0.1.jar
 kerby-xdr-1.0.1.jar
 kryo-shaded-4.0.2.jar
-kubernetes-client-4.4.2.jar
-kubernetes-model-4.4.2.jar
-kubernetes-model-common-4.4.2.jar
+kubernetes-client-4.6.1.jar
+kubernetes-model-4.6.1.jar
+kubernetes-model-common-4.6.1.jar
 leveldbjni-all-1.8.jar
 libfb303-0.9.3.jar
 libthrift-0.9.3.jar
@@ -171,8 +170,8 @@ netty-all-4.1.42.Final.jar
 nimbus-jose-jwt-4.41.1.jar
 objenesis-2.5.1.jar
 okhttp-2.7.5.jar
-okhttp-3.8.1.jar
-okio-1.13.0.jar
+okhttp-3.12.0.jar
+okio-1.15.0.jar
 opencsv-2.3.jar
 orc-core-1.5.5-nohive.jar
 orc-mapreduce-1.5.5-nohive.jar

diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
@@ -29,7 +29,7 @@
   <name>Spark Project Kubernetes</name>
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
-    <kubernetes.client.version>4.4.2</kubernetes.client.version>
+    <kubernetes.client.version>4.6.1</kubernetes.client.version>
   </properties>
 
   <dependencies>
@@ -83,12 +83,6 @@
     </dependency>
     <!-- End of shaded deps. -->
 
-    <dependency>
-      <groupId>com.squareup.okhttp3</groupId>
-      <artifactId>okhttp</artifactId>
-      <version>3.8.1</version>
-    </dependency>
-
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -29,7 +29,7 @@
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
     <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
-    <kubernetes-client.version>4.4.2</kubernetes-client.version>
+    <kubernetes-client.version>4.6.1</kubernetes-client.version>
     <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
     <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>