Updated keyboard interrupt warning (#768)

Updated keyboard interrupt warning to warn for straggling instances on multiple keyboard interrupts.
skyplane-project · Mar 4, 2023 · 9757700 · 9757700
1 parent 25539e1
commit 9757700
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 6 deletions.
diff --git a/skyplane/cli/cli_transfer.py b/skyplane/cli/cli_transfer.py
@@ -252,7 +252,7 @@ def estimate_small_transfer(self, dp: skyplane.Dataplane, size_threshold_bytes:
 
 def force_deprovision(dp: skyplane.Dataplane):
     s = signal.signal(signal.SIGINT, signal.SIG_IGN)
-    dp.deprovision()
+    dp.deprovision(debug=True)
     signal.signal(signal.SIGINT, s)
 
 
@@ -354,10 +354,19 @@ def cp(
                 dp.provision(spinner=True)
                 dp.run(ProgressBarTransferHook())
             except KeyboardInterrupt:
-                logger.fs.warning("Transfer cancelled by user (KeyboardInterrupt)")
-                console.print("\n[bold red]Transfer cancelled by user. Copying gateway logs and exiting.[/bold red]")
+                logger.fs.warning("Transfer cancelled by user (KeyboardInterrupt).")
+                console.print("\n[red]Transfer cancelled by user. Copying gateway logs and exiting.[/red]")
                 do_parallel(dp.copy_log, dp.bound_nodes.values(), n=-1)
-                force_deprovision(dp)
+                try:
+                    force_deprovision(dp)
+                except Exception as e:
+                    logger.fs.exception(e)
+                    console.print(f"[bright_black]{traceback.format_exc()}[/bright_black]")
+                    console.print(e)
+                    UsageClient.log_exception("cli_cp", e, args, cli.src_region_tag, cli.dst_region_tag)
+                    console.print("[bold red]Deprovisioning was interrupted! VMs may still be running which will incur charges.[/bold red]")
+                    console.print("[bold red]Please manually deprovision the VMs by running `skyplane deprovision`.[/bold red]")
+                return 1
             except skyplane.exceptions.SkyplaneException as e:
                 console.print(f"[bright_black]{traceback.format_exc()}[/bright_black]")
                 console.print(e.pretty_print_str())

diff --git a/skyplane/compute/gcp/gcp_auth.py b/skyplane/compute/gcp/gcp_auth.py
@@ -142,7 +142,7 @@ def get_service_account_key(self, service_account_email):
             keys = service.projects().serviceAccounts().keys().list(name="projects/-/serviceAccounts/" + service_account_email).execute()
 
             # cannot have more than 10 keys per service account
-            if len(keys["keys"]) >= 10: 
+            if len(keys["keys"]) >= 10:
                 logger.warning(f"Service account {service_account_email} has too many keys. Deleting stale keys to create new key.")
                 deleted_keys = 0
                 for key in keys["keys"]:

diff --git a/skyplane/compute/server.py b/skyplane/compute/server.py
@@ -312,7 +312,6 @@ def check_stderr(tup):
 
         # copy service account files
         if self.provider == "gcp":
-
             service_key_path = compute.GCPAuthentication().get_service_account_key_path()
             service_key_file = os.path.basename(service_key_path)
             self.upload_file(service_key_path, f"/tmp/{service_key_file}")