From 975770077321b87778df2396d419b8905cf52c5f Mon Sep 17 00:00:00 2001 From: Asim Biswal Date: Fri, 3 Mar 2023 19:04:27 -0800 Subject: [PATCH] Updated keyboard interrupt warning (#768) Updated keyboard interrupt warning to warn for straggling instances on multiple keyboard interrupts. --- skyplane/cli/cli_transfer.py | 17 +++++++++++++---- skyplane/compute/gcp/gcp_auth.py | 2 +- skyplane/compute/server.py | 1 - 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/skyplane/cli/cli_transfer.py b/skyplane/cli/cli_transfer.py index f5e1912f8..69d98cafe 100644 --- a/skyplane/cli/cli_transfer.py +++ b/skyplane/cli/cli_transfer.py @@ -252,7 +252,7 @@ def estimate_small_transfer(self, dp: skyplane.Dataplane, size_threshold_bytes: def force_deprovision(dp: skyplane.Dataplane): s = signal.signal(signal.SIGINT, signal.SIG_IGN) - dp.deprovision() + dp.deprovision(debug=True) signal.signal(signal.SIGINT, s) @@ -354,10 +354,19 @@ def cp( dp.provision(spinner=True) dp.run(ProgressBarTransferHook()) except KeyboardInterrupt: - logger.fs.warning("Transfer cancelled by user (KeyboardInterrupt)") - console.print("\n[bold red]Transfer cancelled by user. Copying gateway logs and exiting.[/bold red]") + logger.fs.warning("Transfer cancelled by user (KeyboardInterrupt).") + console.print("\n[red]Transfer cancelled by user. Copying gateway logs and exiting.[/red]") do_parallel(dp.copy_log, dp.bound_nodes.values(), n=-1) - force_deprovision(dp) + try: + force_deprovision(dp) + except Exception as e: + logger.fs.exception(e) + console.print(f"[bright_black]{traceback.format_exc()}[/bright_black]") + console.print(e) + UsageClient.log_exception("cli_cp", e, args, cli.src_region_tag, cli.dst_region_tag) + console.print("[bold red]Deprovisioning was interrupted! VMs may still be running which will incur charges.[/bold red]") + console.print("[bold red]Please manually deprovision the VMs by running `skyplane deprovision`.[/bold red]") + return 1 except skyplane.exceptions.SkyplaneException as e: console.print(f"[bright_black]{traceback.format_exc()}[/bright_black]") console.print(e.pretty_print_str()) diff --git a/skyplane/compute/gcp/gcp_auth.py b/skyplane/compute/gcp/gcp_auth.py index 053039270..688dcde89 100644 --- a/skyplane/compute/gcp/gcp_auth.py +++ b/skyplane/compute/gcp/gcp_auth.py @@ -142,7 +142,7 @@ def get_service_account_key(self, service_account_email): keys = service.projects().serviceAccounts().keys().list(name="projects/-/serviceAccounts/" + service_account_email).execute() # cannot have more than 10 keys per service account - if len(keys["keys"]) >= 10: + if len(keys["keys"]) >= 10: logger.warning(f"Service account {service_account_email} has too many keys. Deleting stale keys to create new key.") deleted_keys = 0 for key in keys["keys"]: diff --git a/skyplane/compute/server.py b/skyplane/compute/server.py index dc94fa065..512bebda5 100644 --- a/skyplane/compute/server.py +++ b/skyplane/compute/server.py @@ -312,7 +312,6 @@ def check_stderr(tup): # copy service account files if self.provider == "gcp": - service_key_path = compute.GCPAuthentication().get_service_account_key_path() service_key_file = os.path.basename(service_key_path) self.upload_file(service_key_path, f"/tmp/{service_key_file}")