[SPARK-27812][K8S] Bump K8S client version to 4.6.1

### What changes were proposed in this pull request?

Updated kubernetes client.

### Why are the changes needed?

https://issues.apache.org/jira/browse/SPARK-27812
https://issues.apache.org/jira/browse/SPARK-27927

We need this fix fabric8io/kubernetes-client#1768 that was released on version 4.6 of the client. The root cause of the problem is better explained in #25785

### Does this PR introduce any user-facing change?

Nope, it should be transparent to users

### How was this patch tested?

This patch was tested manually using a simple pyspark job

```python
from pyspark.sql import SparkSession

if __name__ == '__main__':
    spark = SparkSession.builder.getOrCreate()
```

The expected behaviour of this "job" is that both python's and jvm's process exit automatically after the main runs. This is the case for spark versions <= 2.4. On version 2.4.3, the jvm process hangs because there's a non daemon thread running

```
"OkHttp WebSocket https://10.96.0.1/..." #121 prio=5 os_prio=0 tid=0x00007fb27c005800 nid=0x24b waiting on condition [0x00007fb300847000]
"OkHttp WebSocket https://10.96.0.1/..." #117 prio=5 os_prio=0 tid=0x00007fb28c004000 nid=0x247 waiting on condition [0x00007fb300e4b000]
```
This is caused by a bug on `kubernetes-client` library, which is fixed on the version that we are upgrading to.

When the mentioned job is run with this patch applied, the behaviour from spark <= 2.4.3 is restored and both processes terminate successfully

Closes #26093 from igorcalabria/k8s-client-update.

Authored-by: igor.calabria <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

dev/deps/spark-deps-hadoop-2.7

@@ -107,7 +107,6 @@ jakarta.ws.rs-api-2.1.5.jar
 jakarta.xml.bind-api-2.3.2.jar
 janino-3.0.15.jar
 javassist-3.22.0-CR2.jar
-javax.el-3.0.1-b11.jar
 javax.inject-1.jar
 javax.servlet-api-3.1.0.jar
 javolution-5.5.1.jar
@@ -138,9 +137,9 @@ jsr305-3.0.0.jar
 jta-1.1.jar
 jul-to-slf4j-1.7.16.jar
 kryo-shaded-4.0.2.jar
-kubernetes-client-4.4.2.jar
-kubernetes-model-4.4.2.jar
-kubernetes-model-common-4.4.2.jar
+kubernetes-client-4.6.1.jar
+kubernetes-model-4.6.1.jar
+kubernetes-model-common-4.6.1.jar
 leveldbjni-all-1.8.jar
 libfb303-0.9.3.jar
 libthrift-0.12.0.jar
@@ -158,8 +157,8 @@ minlog-1.3.0.jar
 netty-all-4.1.42.Final.jar
 objenesis-2.5.1.jar
 okapi-shade-0.4.2.jar
-okhttp-3.8.1.jar
-okio-1.13.0.jar
+okhttp-3.12.0.jar
+okio-1.15.0.jar
 opencsv-2.3.jar
 orc-core-1.5.6-nohive.jar
 orc-mapreduce-1.5.6-nohive.jar

dev/deps/spark-deps-hadoop-3.2

@@ -120,7 +120,6 @@ jakarta.ws.rs-api-2.1.5.jar
 jakarta.xml.bind-api-2.3.2.jar
 janino-3.0.15.jar
 javassist-3.22.0-CR2.jar
-javax.el-3.0.1-b11.jar
 javax.inject-1.jar
 javax.jdo-3.2.0-m3.jar
 javax.servlet-api-3.1.0.jar
@@ -168,9 +167,9 @@ kerby-pkix-1.0.1.jar
 kerby-util-1.0.1.jar
 kerby-xdr-1.0.1.jar
 kryo-shaded-4.0.2.jar
-kubernetes-client-4.4.2.jar
-kubernetes-model-4.4.2.jar
-kubernetes-model-common-4.4.2.jar
+kubernetes-client-4.6.1.jar
+kubernetes-model-4.6.1.jar
+kubernetes-model-common-4.6.1.jar
 leveldbjni-all-1.8.jar
 libfb303-0.9.3.jar
 libthrift-0.12.0.jar
@@ -191,8 +190,8 @@ nimbus-jose-jwt-4.41.1.jar
 objenesis-2.5.1.jar
 okapi-shade-0.4.2.jar
 okhttp-2.7.5.jar
-okhttp-3.8.1.jar
-okio-1.13.0.jar
+okhttp-3.12.0.jar
+okio-1.15.0.jar
 opencsv-2.3.jar
 orc-core-1.5.6.jar
 orc-mapreduce-1.5.6.jar

resource-managers/kubernetes/core/pom.xml

@@ -29,7 +29,7 @@
   <name>Spark Project Kubernetes</name>
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
-    <kubernetes.client.version>4.4.2</kubernetes.client.version>
+    <kubernetes.client.version>4.6.1</kubernetes.client.version>
   </properties>
 
   <dependencies>
@@ -88,12 +88,6 @@
     </dependency>
     <!-- End of shaded deps. -->
 
-    <dependency>
-      <groupId>com.squareup.okhttp3</groupId>
-      <artifactId>okhttp</artifactId>
-      <version>3.8.1</version>
-    </dependency>
-
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

resource-managers/kubernetes/integration-tests/pom.xml

@@ -29,7 +29,7 @@
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
     <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
-    <kubernetes-client.version>4.4.2</kubernetes-client.version>
+    <kubernetes-client.version>4.6.1</kubernetes-client.version>
     <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
     <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>