Upgrade kubernetes-client to 5.12.0

dev/deps/spark-deps-hadoop-2-hive-2.3

@@ -118,7 +118,7 @@ jackson-core/2.13.1//jackson-core-2.13.1.jar
 jackson-databind/2.13.1//jackson-databind-2.13.1.jar
 jackson-dataformat-cbor/2.13.1//jackson-dataformat-cbor-2.13.1.jar
 jackson-dataformat-yaml/2.13.1//jackson-dataformat-yaml-2.13.1.jar
-jackson-datatype-jsr310/2.13.0//jackson-datatype-jsr310-2.13.0.jar
+jackson-datatype-jsr310/2.13.1//jackson-datatype-jsr310-2.13.1.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-scala_2.12/2.13.1//jackson-module-scala_2.12-2.13.1.jar
@@ -162,27 +162,27 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.32//jul-to-slf4j-1.7.32.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/5.10.2//kubernetes-client-5.10.2.jar
-kubernetes-model-admissionregistration/5.10.2//kubernetes-model-admissionregistration-5.10.2.jar
-kubernetes-model-apiextensions/5.10.2//kubernetes-model-apiextensions-5.10.2.jar
-kubernetes-model-apps/5.10.2//kubernetes-model-apps-5.10.2.jar
-kubernetes-model-autoscaling/5.10.2//kubernetes-model-autoscaling-5.10.2.jar
-kubernetes-model-batch/5.10.2//kubernetes-model-batch-5.10.2.jar
-kubernetes-model-certificates/5.10.2//kubernetes-model-certificates-5.10.2.jar
-kubernetes-model-common/5.10.2//kubernetes-model-common-5.10.2.jar
-kubernetes-model-coordination/5.10.2//kubernetes-model-coordination-5.10.2.jar
-kubernetes-model-core/5.10.2//kubernetes-model-core-5.10.2.jar
-kubernetes-model-discovery/5.10.2//kubernetes-model-discovery-5.10.2.jar
-kubernetes-model-events/5.10.2//kubernetes-model-events-5.10.2.jar
-kubernetes-model-extensions/5.10.2//kubernetes-model-extensions-5.10.2.jar
-kubernetes-model-flowcontrol/5.10.2//kubernetes-model-flowcontrol-5.10.2.jar
-kubernetes-model-metrics/5.10.2//kubernetes-model-metrics-5.10.2.jar
-kubernetes-model-networking/5.10.2//kubernetes-model-networking-5.10.2.jar
-kubernetes-model-node/5.10.2//kubernetes-model-node-5.10.2.jar
-kubernetes-model-policy/5.10.2//kubernetes-model-policy-5.10.2.jar
-kubernetes-model-rbac/5.10.2//kubernetes-model-rbac-5.10.2.jar
-kubernetes-model-scheduling/5.10.2//kubernetes-model-scheduling-5.10.2.jar
-kubernetes-model-storageclass/5.10.2//kubernetes-model-storageclass-5.10.2.jar
+kubernetes-client/5.12.0//kubernetes-client-5.12.0.jar
+kubernetes-model-admissionregistration/5.12.0//kubernetes-model-admissionregistration-5.12.0.jar
+kubernetes-model-apiextensions/5.12.0//kubernetes-model-apiextensions-5.12.0.jar
+kubernetes-model-apps/5.12.0//kubernetes-model-apps-5.12.0.jar
+kubernetes-model-autoscaling/5.12.0//kubernetes-model-autoscaling-5.12.0.jar
+kubernetes-model-batch/5.12.0//kubernetes-model-batch-5.12.0.jar
+kubernetes-model-certificates/5.12.0//kubernetes-model-certificates-5.12.0.jar
+kubernetes-model-common/5.12.0//kubernetes-model-common-5.12.0.jar
+kubernetes-model-coordination/5.12.0//kubernetes-model-coordination-5.12.0.jar
+kubernetes-model-core/5.12.0//kubernetes-model-core-5.12.0.jar
+kubernetes-model-discovery/5.12.0//kubernetes-model-discovery-5.12.0.jar
+kubernetes-model-events/5.12.0//kubernetes-model-events-5.12.0.jar
+kubernetes-model-extensions/5.12.0//kubernetes-model-extensions-5.12.0.jar
+kubernetes-model-flowcontrol/5.12.0//kubernetes-model-flowcontrol-5.12.0.jar
+kubernetes-model-metrics/5.12.0//kubernetes-model-metrics-5.12.0.jar
+kubernetes-model-networking/5.12.0//kubernetes-model-networking-5.12.0.jar
+kubernetes-model-node/5.12.0//kubernetes-model-node-5.12.0.jar
+kubernetes-model-policy/5.12.0//kubernetes-model-policy-5.12.0.jar
+kubernetes-model-rbac/5.12.0//kubernetes-model-rbac-5.12.0.jar
+kubernetes-model-scheduling/5.12.0//kubernetes-model-scheduling-5.12.0.jar
+kubernetes-model-storageclass/5.12.0//kubernetes-model-storageclass-5.12.0.jar
 lapack/2.2.1//lapack-2.2.1.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -109,7 +109,7 @@ jackson-core/2.13.1//jackson-core-2.13.1.jar
 jackson-databind/2.13.1//jackson-databind-2.13.1.jar
 jackson-dataformat-cbor/2.13.1//jackson-dataformat-cbor-2.13.1.jar
 jackson-dataformat-yaml/2.13.1//jackson-dataformat-yaml-2.13.1.jar
-jackson-datatype-jsr310/2.13.0//jackson-datatype-jsr310-2.13.0.jar
+jackson-datatype-jsr310/2.13.1//jackson-datatype-jsr310-2.13.1.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-scala_2.12/2.13.1//jackson-module-scala_2.12-2.13.1.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
@@ -148,27 +148,27 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.32//jul-to-slf4j-1.7.32.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/5.10.2//kubernetes-client-5.10.2.jar
-kubernetes-model-admissionregistration/5.10.2//kubernetes-model-admissionregistration-5.10.2.jar
-kubernetes-model-apiextensions/5.10.2//kubernetes-model-apiextensions-5.10.2.jar
-kubernetes-model-apps/5.10.2//kubernetes-model-apps-5.10.2.jar
-kubernetes-model-autoscaling/5.10.2//kubernetes-model-autoscaling-5.10.2.jar
-kubernetes-model-batch/5.10.2//kubernetes-model-batch-5.10.2.jar
-kubernetes-model-certificates/5.10.2//kubernetes-model-certificates-5.10.2.jar
-kubernetes-model-common/5.10.2//kubernetes-model-common-5.10.2.jar
-kubernetes-model-coordination/5.10.2//kubernetes-model-coordination-5.10.2.jar
-kubernetes-model-core/5.10.2//kubernetes-model-core-5.10.2.jar
-kubernetes-model-discovery/5.10.2//kubernetes-model-discovery-5.10.2.jar
-kubernetes-model-events/5.10.2//kubernetes-model-events-5.10.2.jar
-kubernetes-model-extensions/5.10.2//kubernetes-model-extensions-5.10.2.jar
-kubernetes-model-flowcontrol/5.10.2//kubernetes-model-flowcontrol-5.10.2.jar
-kubernetes-model-metrics/5.10.2//kubernetes-model-metrics-5.10.2.jar
-kubernetes-model-networking/5.10.2//kubernetes-model-networking-5.10.2.jar
-kubernetes-model-node/5.10.2//kubernetes-model-node-5.10.2.jar
-kubernetes-model-policy/5.10.2//kubernetes-model-policy-5.10.2.jar
-kubernetes-model-rbac/5.10.2//kubernetes-model-rbac-5.10.2.jar
-kubernetes-model-scheduling/5.10.2//kubernetes-model-scheduling-5.10.2.jar
-kubernetes-model-storageclass/5.10.2//kubernetes-model-storageclass-5.10.2.jar
+kubernetes-client/5.12.0//kubernetes-client-5.12.0.jar
+kubernetes-model-admissionregistration/5.12.0//kubernetes-model-admissionregistration-5.12.0.jar
+kubernetes-model-apiextensions/5.12.0//kubernetes-model-apiextensions-5.12.0.jar
+kubernetes-model-apps/5.12.0//kubernetes-model-apps-5.12.0.jar
+kubernetes-model-autoscaling/5.12.0//kubernetes-model-autoscaling-5.12.0.jar
+kubernetes-model-batch/5.12.0//kubernetes-model-batch-5.12.0.jar
+kubernetes-model-certificates/5.12.0//kubernetes-model-certificates-5.12.0.jar
+kubernetes-model-common/5.12.0//kubernetes-model-common-5.12.0.jar
+kubernetes-model-coordination/5.12.0//kubernetes-model-coordination-5.12.0.jar
+kubernetes-model-core/5.12.0//kubernetes-model-core-5.12.0.jar
+kubernetes-model-discovery/5.12.0//kubernetes-model-discovery-5.12.0.jar
+kubernetes-model-events/5.12.0//kubernetes-model-events-5.12.0.jar
+kubernetes-model-extensions/5.12.0//kubernetes-model-extensions-5.12.0.jar
+kubernetes-model-flowcontrol/5.12.0//kubernetes-model-flowcontrol-5.12.0.jar
+kubernetes-model-metrics/5.12.0//kubernetes-model-metrics-5.12.0.jar
+kubernetes-model-networking/5.12.0//kubernetes-model-networking-5.12.0.jar
+kubernetes-model-node/5.12.0//kubernetes-model-node-5.12.0.jar
+kubernetes-model-policy/5.12.0//kubernetes-model-policy-5.12.0.jar
+kubernetes-model-rbac/5.12.0//kubernetes-model-rbac-5.12.0.jar
+kubernetes-model-scheduling/5.12.0//kubernetes-model-scheduling-5.12.0.jar
+kubernetes-model-storageclass/5.12.0//kubernetes-model-storageclass-5.12.0.jar
 lapack/2.2.1//lapack-2.2.1.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar

pom.xml

@@ -204,7 +204,7 @@
     <arrow.version>6.0.1</arrow.version>
     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
-    <kubernetes-client.version>5.10.2</kubernetes-client.version>
+    <kubernetes-client.version>5.12.0</kubernetes-client.version>
 
     <test.java.home>${java.home}</test.java.home>
 

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala

@@ -24,9 +24,10 @@ import com.google.common.io.Files
 import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient, KubernetesClient}
 import io.fabric8.kubernetes.client.Config.KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY
 import io.fabric8.kubernetes.client.Config.autoConfigure
-import io.fabric8.kubernetes.client.utils.HttpClientUtils
+import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
 import io.fabric8.kubernetes.client.utils.Utils.getSystemPropertyOrEnvVar
 import okhttp3.Dispatcher
+import okhttp3.OkHttpClient
 
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.k8s.Config._
@@ -68,6 +69,8 @@ private[spark] object SparkKubernetesClientFactory extends Logging {
       .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_KEY_FILE_CONF_SUFFIX")
     val clientCertFile = sparkConf
       .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_CERT_FILE_CONF_SUFFIX")
+    // TODO(SPARK-37687): clean up direct usage of OkHttpClient, see also:
+    // https://github.com/fabric8io/kubernetes-client/issues/3547
     val dispatcher = new Dispatcher(
       ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
 
@@ -105,13 +108,14 @@ private[spark] object SparkKubernetesClientFactory extends Logging {
       }.withOption(namespace) {
         (ns, configBuilder) => configBuilder.withNamespace(ns)
       }.build()
-    val baseHttpClient = HttpClientUtils.createHttpClient(config)
-    val httpClientWithCustomDispatcher = baseHttpClient.newBuilder()
-      .dispatcher(dispatcher)
-      .build()
+    val factoryWithCustomDispatcher = new OkHttpClientFactory() {
+      override protected def additionalConfig(builder: OkHttpClient.Builder): Unit = {
+        builder.dispatcher(dispatcher)
+      }
+    }
     logDebug("Kubernetes client config: " +
       new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
-    new DefaultKubernetesClient(httpClientWithCustomDispatcher, config)
+    new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config)
   }
 
   private implicit class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder)

resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala

@@ -24,7 +24,7 @@ import java.util.zip.{ZipEntry, ZipOutputStream}
 import scala.collection.JavaConverters._
 
 import io.fabric8.kubernetes.client.dsl.ExecListener
-import okhttp3.Response
+import io.fabric8.kubernetes.client.dsl.ExecListener.Response
 import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
 import org.apache.commons.compress.utils.IOUtils
@@ -62,7 +62,7 @@ object Utils extends Logging {
       val openLatch: CountDownLatch = new CountDownLatch(1)
       val closeLatch: CountDownLatch = new CountDownLatch(1)
 
-      override def onOpen(response: Response): Unit = {
+      override def onOpen(): Unit = {
         openLatch.countDown()
       }