Skip to content

Commit

Permalink
connect_direct yt_proxy should have a default
Browse files Browse the repository at this point in the history
No description

---

Pull Request resolved: #29
commit_hash:d681414d5faaadc96c148eef8244008113dbc133
  • Loading branch information
faucct authored and robot-piglet committed Sep 26, 2024
1 parent 5c1760a commit 645ec51
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions spyt-package/src/main/python/spyt/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
require_yt_client()

from yt.wrapper import YtClient # noqa: E402
from yt.wrapper.http_helpers import get_token, get_user_name # noqa: E402
from yt.wrapper.http_helpers import get_token, get_user_name, get_proxy_address_netloc # noqa: E402

from .arcadia import checked_extract_spark # noqa: E402
from .utils import default_token, default_discovery_dir, get_spark_master, set_conf, \
Expand Down Expand Up @@ -341,7 +341,7 @@ def connect(num_executors=5,
return spark


def connect_direct(yt_proxy: str, conf: SparkConf = None):
def connect_direct(yt_proxy: str = None, conf: SparkConf = None):
checked_extract_spark()
_set_spark_conf_dir()

Expand All @@ -350,7 +350,9 @@ def connect_direct(yt_proxy: str, conf: SparkConf = None):
conf.set("spark.ytsaurus.python.version", f"{sys.version_info.major}.{sys.version_info.minor}")
if conf.contains("spark.executor.resource.gpu.amount"):
conf.set("spark.executor.resource.gpu.discoveryScript", "./spyt-package/bin/getGpusResources.sh")
spark = SparkSession.builder.config(conf=conf).master("ytsaurus://" + yt_proxy).getOrCreate()
spark = SparkSession.builder.config(conf=conf).master(
"ytsaurus://" + (yt_proxy or get_proxy_address_netloc())
).getOrCreate()
return spark


Expand Down

0 comments on commit 645ec51

Please sign in to comment.