Merge pull request iterative#2160 from efiop/repo

dvc: replace pkg concept with external repo

dvc/cli.py

@@ -9,7 +9,8 @@
 
 from dvc.command.base import fix_subparsers
 import dvc.command.init as init
-import dvc.command.pkg as pkg
+import dvc.command.get as get
+import dvc.command.get_url as get_url
 import dvc.command.destroy as destroy
 import dvc.command.remove as remove
 import dvc.command.move as move
@@ -20,6 +21,7 @@
 import dvc.command.gc as gc
 import dvc.command.add as add
 import dvc.command.imp as imp
+import dvc.command.imp_url as imp_url
 import dvc.command.config as config
 import dvc.command.checkout as checkout
 import dvc.command.remote as remote
@@ -41,7 +43,8 @@
 
 COMMANDS = [
     init,
-    pkg,
+    get,
+    get_url,
     destroy,
     add,
     remove,
@@ -52,6 +55,7 @@
     data_sync,
     gc,
     imp,
+    imp_url,
     config,
     checkout,
     remote,

dvc/command/get.py

@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.repo import Repo
+from dvc.exceptions import DvcException
+from .base import CmdBaseNoRepo, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGet(CmdBaseNoRepo):
+    def run(self):
+        try:
+            Repo.get(
+                self.args.url,
+                path=self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
+            )
+            return 0
+        except DvcException:
+            logger.exception(
+                "failed to get '{}' from '{}'".format(
+                    self.args.path, self.args.url
+                )
+            )
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download or copy files from URL."
+    get_parser = subparsers.add_parser(
+        "get",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="DVC repository URL to download data from."
+    )
+    get_parser.add_argument("path", help="Path to data within DVC repository.")
+    get_parser.add_argument(
+        "-o", "--out", nargs="?", help="Destination path to put data to."
+    )
+    get_parser.add_argument(
+        "--rev", nargs="?", help="DVC repository git revision."
+    )
+    get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -0,0 +1,39 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.repo import Repo
+from dvc.exceptions import DvcException
+from .base import CmdBaseNoRepo, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdGetUrl(CmdBaseNoRepo):
+    def run(self):
+        try:
+            Repo.get_url(self.args.url, out=self.args.out)
+            return 0
+        except DvcException:
+            logger.exception("failed to get '{}'".format(self.args.url))
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    GET_HELP = "Download or copy files from URL."
+    get_parser = subparsers.add_parser(
+        "get-url",
+        parents=[parent_parser],
+        description=append_doc_link(GET_HELP, "get-url"),
+        help=GET_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    get_parser.add_argument(
+        "url", help="See `dvc import-url -h` for full list of supported URLs."
+    )
+    get_parser.add_argument(
+        "out", nargs="?", help="Destination path to put data to."
+    )
+    get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 import argparse
-import os
 import logging
 
-from dvc.utils.compat import urlparse
 from dvc.exceptions import DvcException
 from dvc.command.base import CmdBase, append_doc_link
 
@@ -15,26 +13,26 @@
 class CmdImport(CmdBase):
     def run(self):
         try:
-            default_out = os.path.basename(urlparse(self.args.url).path)
-
-            out = self.args.out or default_out
-
             self.repo.imp(
-                self.args.url, out, self.args.resume, fname=self.args.file
+                self.args.url,
+                path=self.args.path,
+                out=self.args.out,
+                rev=self.args.rev,
             )
         except DvcException:
             logger.exception(
-                "failed to import {}. You could also try downloading "
-                "it manually and adding it with `dvc add` command.".format(
-                    self.args.url
+                "failed to import '{}' from '{}'.".format(
+                    self.args.path, self.args.url
                 )
             )
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+    IMPORT_HELP = (
+        "Download data from DVC repository and take it under DVC control."
+    )
 
     import_parser = subparsers.add_parser(
         "import",
@@ -43,28 +41,14 @@ def add_parser(subparsers, parent_parser):
         help=IMPORT_HELP,
         formatter_class=argparse.RawTextHelpFormatter,
     )
+    import_parser.add_argument("url", help="DVC repository URL.")
     import_parser.add_argument(
-        "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "C:\\\\path\\to\\file\n"
-        "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "gs://bucket/path/to/file\n"
-        "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
-    )
-    import_parser.add_argument(
-        "--resume",
-        action="store_true",
-        default=False,
-        help="Resume previously started download.",
+        "path", nargs="?", help="Path to data within DVC repository."
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
+        "-o", "--out", nargs="?", help="Destination path to put data to."
     )
     import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--rev", nargs="?", help="DVC repository git revision."
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -0,0 +1,67 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdImportUrl(CmdBase):
+    def run(self):
+        try:
+            self.repo.imp_url(
+                self.args.url,
+                out=self.args.out,
+                resume=self.args.resume,
+                fname=self.args.file,
+            )
+        except DvcException:
+            logger.exception(
+                "failed to import {}. You could also try downloading "
+                "it manually and adding it with `dvc add` command.".format(
+                    self.args.url
+                )
+            )
+            return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    IMPORT_HELP = "Download or copy files from URL and take under DVC control."
+
+    import_parser = subparsers.add_parser(
+        "import-url",
+        parents=[parent_parser],
+        description=append_doc_link(IMPORT_HELP, "import-url"),
+        help=IMPORT_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    import_parser.add_argument(
+        "url",
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "C:\\\\path\\to\\file\n"
+        "https://example.com/path/to/file\n"
+        "s3://bucket/path/to/file\n"
+        "gs://bucket/path/to/file\n"
+        "hdfs://example.com/path/to/file\n"
+        "ssh://example.com:/path/to/file\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
+    )
+    import_parser.add_argument(
+        "--resume",
+        action="store_true",
+        default=False,
+        help="Resume previously started download.",
+    )
+    import_parser.add_argument(
+        "out", nargs="?", help="Destination path to put files to."
+    )
+    import_parser.add_argument(
+        "-f", "--file", help="Specify name of the DVC-file it generates."
+    )
+    import_parser.set_defaults(func=CmdImportUrl)