Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deprecate local transfers #430

Merged
merged 2 commits into from
Jun 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/quickstart.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
*********
***************
Getting Started
*********
***************

Installation
-----------------------
Expand Down
65 changes: 11 additions & 54 deletions skyplane/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@
import skyplane.cli.experiments
from skyplane import config_path, exceptions, skyplane_root, cloud_config
from skyplane.cli.common import print_header, console
from skyplane.cli.cli_impl.cp_local import (
copy_azure_local,
copy_gcs_local,
copy_local_azure,
copy_local_gcs,
copy_local_local,
copy_local_s3,
copy_s3_local,
)
from skyplane.cli.cli_impl.cp_replicate import (
generate_full_transferobjlist,
generate_topology,
Expand Down Expand Up @@ -52,33 +43,6 @@
app.add_typer(skyplane.cli.cli_solver.app, name="solver")


@app.command()
def ls(directory: str):
"""
It takes a directory path, parses it, and then calls the appropriate function to list the contents
of that directory. If the path is on an object store, it will list the contents of the object store
at that prefix.

:param directory: str
:type directory: str
"""
provider, bucket, key = parse_path(directory)
if provider == "local":
for path in ls_local(Path(directory)):
typer.echo(path)
elif provider == "s3":
for path in ls_objstore("aws:infer", bucket, key):
typer.echo(path)
elif provider == "gs":
for path in ls_objstore("gcp:infer", bucket, key):
typer.echo(path)
elif provider == "azure":
for path in ls_objstore("azure:infer", bucket, key):
typer.echo(path)
else:
raise NotImplementedError(f"Unrecognized object store provider")


@app.command()
def cp(
src: str,
Expand Down Expand Up @@ -136,26 +100,19 @@ def cp(

clouds = {"s3": "aws:infer", "gs": "gcp:infer", "azure": "azure:infer"}

# error for local transfers
def error_local():
typer.secho("Local transfers are not yet supported (but will be soon!)", fg="red")
typer.secho("Skyplane is currently most optimized for cloud to cloud transfers.", fg="yellow")
typer.secho(
"Please provide feedback for on prem transfers at: https://github.com/skyplane-project/skyplane/discussions/424", fg="yellow"
)
raise typer.Exit(code=1)

# raise file limits for local transfers
if provider_src == "local" or provider_dst == "local":
check_ulimit()
if provider_src == "local" and provider_dst == "local":
copy_local_local(Path(path_src), Path(path_dst))
elif provider_src == "local" and provider_dst == "s3":
copy_local_s3(Path(path_src), bucket_dst, path_dst)
elif provider_src == "s3" and provider_dst == "local":
copy_s3_local(bucket_src, path_src, Path(path_dst))
elif provider_src == "local" and provider_dst == "gs":
copy_local_gcs(Path(path_src), bucket_dst, path_dst)
elif provider_src == "gs" and provider_dst == "local":
copy_gcs_local(bucket_src, path_src, Path(path_dst))
elif provider_src == "local" and provider_dst == "azure":
account_name, container_name = bucket_dst
copy_local_azure(Path(path_src), account_name, container_name, path_dst)
elif provider_src == "azure" and provider_dst == "local":
account_name, container_name = bucket_dst
copy_azure_local(account_name, container_name, path_src, Path(path_dst))
elif provider_src in clouds and provider_dst in clouds:
error_local()
if provider_src in clouds and provider_dst in clouds:
try:
src_client = ObjectStoreInterface.create(clouds[provider_src], bucket_src)
src_region = src_client.region_tag()
Expand Down
131 changes: 0 additions & 131 deletions skyplane/cli/cli_impl/cp_local.py

This file was deleted.