Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix stop standalone #2715

Merged
merged 2 commits into from
Aug 12, 2020
Merged

Fix stop standalone #2715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c97bc28
fix
hkvision Aug 12, 2020
35c0cba
update
hkvision Aug 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pyzoo/zoo/common/nncontext.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def init_spark_standalone(num_executors,
executor_memory="10g",
driver_memory="1g",
driver_cores=4,
spark_master=None,
master=None,
extra_executor_memory_for_ray=None,
extra_python_lib=None,
spark_log_level="WARN",
Expand All @@ -142,8 +142,8 @@ def init_spark_standalone(num_executors,
:param executor_memory: The memory for each executor. Default to be '2g'.
:param driver_cores: The number of cores for the Spark driver. Default to be 4.
:param driver_memory: The memory for the Spark driver. Default to be '1g'.
:param spark_master: The master URL of an existing Spark standalone cluster starting with
'spark://'. You only need to specify this if you have already started a standalone cluster.
:param master: The master URL of an existing Spark standalone cluster: 'spark://master:port'.
You only need to specify this if you have already started a standalone cluster.
Default to be None and a new standalone cluster would be started in this case.
:param extra_executor_memory_for_ray: The extra memory for Ray services. Default to be None.
:param extra_python_lib: Extra python files or packages needed for distribution.
Expand All @@ -167,7 +167,7 @@ def init_spark_standalone(num_executors,
executor_memory=executor_memory,
driver_memory=driver_memory,
driver_cores=driver_cores,
spark_master=spark_master,
master=master,
extra_executor_memory_for_ray=extra_executor_memory_for_ray,
extra_python_lib=extra_python_lib,
conf=conf,
Expand Down
17 changes: 10 additions & 7 deletions pyzoo/zoo/util/spark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ def init_spark_standalone(self,
executor_memory="10g",
driver_memory="1g",
driver_cores=4,
spark_master=None,
master=None,
extra_executor_memory_for_ray=None,
extra_python_lib=None,
conf=None,
Expand All @@ -248,7 +248,7 @@ def init_spark_standalone(self,

if 'PYSPARK_PYTHON' not in os.environ:
os.environ["PYSPARK_PYTHON"] = self._detect_python_location()
if not spark_master:
if not master:
pyspark_home = os.path.abspath(pyspark.__file__ + "/../")
zoo_standalone_home = os.path.abspath(__file__ + "/../../share/bin/standalone")
node_ip = get_node_ip()
Expand All @@ -266,18 +266,19 @@ def init_spark_standalone(self,
"{}/sbin/start-master.sh".format(zoo_standalone_home),
shell=True, env=SparkRunner.standalone_env)
os.waitpid(start_master_pro.pid, 0)
spark_master = "spark://{}:7077".format(node_ip) # 7077 is the default port
master = "spark://{}:7077".format(node_ip) # 7077 is the default port
# Start worker
start_worker_pro = subprocess.Popen(
"{}/sbin/start-worker.sh {}".format(zoo_standalone_home, spark_master),
"{}/sbin/start-worker.sh {}".format(zoo_standalone_home, master),
shell=True, env=SparkRunner.standalone_env)
os.waitpid(start_worker_pro.pid, 0)
else: # A Spark standalone cluster has already been started by the user.
assert spark_master.startswith("spark://"), \
"Please input a valid master address for Spark standalone: spark://master:port"
assert master.startswith("spark://"), \
"Please input a valid master address for your Spark standalone cluster: " \
"spark://master:port"

# Start pyspark-shell
submit_args = " --master " + spark_master
submit_args = " --master " + master
submit_args = submit_args + " --driver-cores {} --driver-memory {} --num-executors {}" \
" --executor-cores {} --executor-memory {}"\
.format(driver_cores, driver_memory, num_executors, executor_cores, executor_memory)
Expand Down Expand Up @@ -318,6 +319,8 @@ def stop_spark_standalone():
import pyspark
pyspark_home = os.path.abspath(pyspark.__file__ + "/../")
zoo_standalone_home = os.path.abspath(__file__ + "/../../share/bin/standalone")
pro = subprocess.Popen(["chmod", "-R", "+x", "{}/sbin".format(zoo_standalone_home)])
os.waitpid(pro.pid, 0)
env = {"SPARK_HOME": pyspark_home,
"ZOO_STANDALONE_HOME": zoo_standalone_home}
stop_worker_pro = subprocess.Popen(
Expand Down