remove more usages of python_cluster_paths

ytsaurus · Oct 22, 2024 · 4b7a534 · 4b7a534
1 parent 963939a
commit 4b7a534
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 19 deletions.
diff --git a/e2e-test/common/cypress.py b/e2e-test/common/cypress.py
@@ -1,6 +1,5 @@
 import logging
 import os
-import subprocess
 
 logger = logging.getLogger(__name__)
 
@@ -23,12 +22,6 @@ def set_spark_conf(options, yt_client):
 
 
 def set_python_path(python_path, yt_client):
-    py_version_full = (subprocess.run([python_path, '-V'], capture_output=True).stdout.decode()
-                       .replace("Python", "").strip())
-    py_version_short = py_version_full[:py_version_full.index('.', py_version_full.index('.') + 1)]
-    logger.info(f"Set python interpreter (version {py_version_short}): {python_path}")
-    python_cluster_paths = {py_version_short: python_path}
-    yt_client.set("//home/spark/conf/global/python_cluster_paths", python_cluster_paths)
     set_spark_conf({'spark.pyspark.python': python_path}, yt_client)
 
 

diff --git a/...src/main/scala/org/apache/spark/scheduler/cluster/ytsaurus/YTsaurusOperationManager.scala b/...src/main/scala/org/apache/spark/scheduler/cluster/ytsaurus/YTsaurusOperationManager.scala
@@ -28,7 +28,6 @@ private[spark] class YTsaurusOperationManager(val ytClient: YTsaurusClient,
                                               user: String, token: String,
                                               portoLayers: YTreeNode,
                                               filePaths: YTreeNode,
-                                              pythonPaths: YTreeMapNode,
                                               environment: YTreeMapNode,
                                               home: String,
                                               prepareEnvCommand: String,
@@ -230,15 +229,7 @@ private[spark] class YTsaurusOperationManager(val ytClient: YTsaurusClient,
     val execEnvironmentBuilder = environment.toMapBuilder
 
     if (isPythonApp && conf.get(YTSAURUS_PYTHON_VERSION).isDefined) {
-      val pythonVersion = conf.get(YTSAURUS_PYTHON_VERSION).get
-      val pythonPathOpt = pythonPaths.get(pythonVersion)
-      if (pythonPathOpt.isEmpty) {
-        throw new SparkException(
-          s"Python version $pythonVersion is not supported. Please check the path specified in " +
-            s"$GLOBAL_CONFIG_PATH for supported versions"
-        )
-      }
-      execEnvironmentBuilder.key("PYSPARK_EXECUTOR_PYTHON").value(pythonPathOpt.get())
+      execEnvironmentBuilder.key("PYSPARK_EXECUTOR_PYTHON").value(f"python${conf.get(YTSAURUS_PYTHON_VERSION).get}")
     }
 
     var executorCommand = (Seq(
@@ -309,7 +300,6 @@ private[spark] object YTsaurusOperationManager extends Logging {
       val portoLayers = getPortoLayers(conf, releaseConfig.getListO("layer_paths").orElse(YTree.listBuilder().buildList()))
 
       val filePaths = releaseConfig.getListO("file_paths").orElse(YTree.listBuilder().buildList())
-      val pythonPaths = globalConfig.getMapO("python_cluster_paths").orElse(YTree.mapBuilder().buildMap())
       val cudaVersion = globalConfig.getStringO("cuda_toolkit_version").orElse("11.0")
 
       applicationFiles(conf).foreach { appFile =>
@@ -365,7 +355,6 @@ private[spark] object YTsaurusOperationManager extends Logging {
         token,
         portoLayers,
         filePaths,
-        pythonPaths,
         environment,
         home,
         prepareEnvCommand,

diff --git a/tools/release/publisher/config_generator.py b/tools/release/publisher/config_generator.py
@@ -144,6 +144,7 @@ def prepare_global_config(os_release: bool) -> Dict[str, Any]:
     global_config['latest_spyt_version'] = "1.76.1"
     global_config['latest_spark_cluster_version'] = "1.75.4"
     if not os_release:
+        # is preserved for backward compatibility and is subject to remove
         python_cluster_paths = {
             "3.11": "/opt/python3.11/bin/python3.11",
             "3.12": "/opt/python3.12/bin/python3.12",