Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[App] Actually fix launching apps on multiple clusters #15461

Closed
wants to merge 2 commits into from
+19 −7
Closed

[App] Actually fix launching apps on multiple clusters #15461

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4460162
Actually fix launching apps on multiple clusters
luca3rd Nov 2, 2022
698669b
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions src/lightning_app/runners/cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ def dispatch(
if not has_sufficient_credits:
logger.warn("You may need Lightning credits to run your apps on the cloud.")

# right now we only allow a single instance of the app
# right now we only allow a single instance of the app per cluster
find_instances_resp = self.backend.client.lightningapp_instance_service_list_lightningapp_instances(
project_id=project.project_id, app_id=lit_app.id
)
Expand All @@ -285,14 +285,25 @@ def dispatch(
elif CLOUD_QUEUE_TYPE == "redis":
queue_server_type = V1QueueServerType.REDIS

if find_instances_resp.lightningapps:
existing_instance = find_instances_resp.lightningapps[0]
previous_app_instances = find_instances_resp.lightningapps
previous_app_instances = sorted(previous_app_instances, key=lambda i: i.created_at, reverse=True)

existing_instances_by_cluster = {}
for instance in previous_app_instances:
if instance.spec.desired_state == V1LightningappInstanceState.RUNNING:
existing_instances_by_cluster[instance.spec.cluster_id] = instance

if previous_app_instances:
most_recent_instance = previous_app_instances[0]
if not app_config.cluster_id:
# Re-run the app on the same cluster
app_config.cluster_id = existing_instance.spec.cluster_id
# Re-run the app on the most recent cluster
app_config.cluster_id = most_recent_instance.spec.cluster_id

# get the existing instance on the cluster, if any
existing_instance = existing_instances_by_cluster.get(app_config.cluster_id, None)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
existing_instance = existing_instances_by_cluster.get(app_config.cluster_id, None)
existing_instance = existing_instances_by_cluster.get(app_config.cluster_id)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

None is already the default returning value :)


if existing_instance.status.phase != V1LightningappInstanceState.STOPPED:
# stop old instances if releasing to the same cluster
if existing_instance and existing_instance.status.phase != V1LightningappInstanceState.STOPPED:
# TODO(yurij): Implement release switching in the UI and remove this
# We can only switch release of the stopped instance
existing_instance = self.backend.client.lightningapp_instance_service_update_lightningapp_instance(
Expand Down Expand Up @@ -367,7 +378,8 @@ def dispatch(
repo.package()
repo.upload(url=lightning_app_release.source_upload_url)

if find_instances_resp.lightningapps:
# updates the app instance if on the same cluster
if existing_instance:
lightning_app_instance = (
self.backend.client.lightningapp_instance_service_update_lightningapp_instance_release(
project_id=project.project_id,
Expand Down