Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

On-prem support for Skyplane #540

Merged
merged 13 commits into from
Sep 15, 2022
37 changes: 28 additions & 9 deletions skyplane/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from shlex import split
import traceback
import uuid
import os

from rich import print as rprint

Expand Down Expand Up @@ -112,6 +113,7 @@ def cp(
provider_dst, bucket_dst, path_dst = parse_path(dst)

clouds = {"s3": "aws:infer", "gs": "gcp:infer", "azure": "azure:infer"}
cloud_provider_api = {"s3": "aws s3 cp", "gs": "gsutil -m cp -r"}

args = {
"cmd": "cp",
Expand All @@ -136,16 +138,33 @@ def cp(

requester_pays: bool = cloud_config.get_flag("requester_pays")

if provider_src == "local" or provider_dst == "local":
typer.secho("Local transfers are not yet supported (but will be soon!)", fg="red", err=True)
typer.secho("Skyplane is currently most optimized for cloud to cloud transfers.", fg="yellow", err=True)
typer.secho(
"Please provide feedback for on prem transfers at: https://github.com/skyplane-project/skyplane/discussions/424",
fg="yellow",
err=True,
if provider_src == "local" and provider_dst == "local":
typer.secho(f"Copying between local paths", fg="yellow")
cmd = "cp -r " + path_src + " " + path_dst
typer.secho(f"Falling back to: {cmd}")
os.system(cmd)

elif provider_src == "local" or provider_dst in clouds:
typer.secho(f"Copying from local to {provider_dst}", fg="yellow")
cmd = (
cloud_provider_api[provider_dst] + " " + path_src + f" s3://{bucket_dst}/{path_dst}" + " --recursive"
if (provider_dst == "s3")
else ""
)
typer.secho(f"Falling back to: {cmd}")
os.system(cmd)

elif provider_src in clouds and provider_dst == "local":
typer.secho(f"Copying from {provider_src} to local", fg="yellow")
cmd = (
cloud_provider_api[provider_src] + f" s3://{bucket_src}/{path_src} " + path_dst + " --recursive"
if (provider_src == "s3")
else ""
)
raise typer.Exit(code=1)
if provider_src in clouds and provider_dst in clouds:
typer.secho(f"Falling back to: {cmd}")
os.system(cmd)

elif provider_src in clouds and provider_dst in clouds:
try:
src_client = ObjectStoreInterface.create(clouds[provider_src], bucket_src)
dst_client = ObjectStoreInterface.create(clouds[provider_dst], bucket_dst)
Expand Down