introduce dvc update for ext repos (#2218)

* erepo: dvc: implement `update` for imports from ext repos

Fixes #1774
Fixes #2139
Fixes #2201

Signed-off-by: Ruslan Kuprieiev <[email protected]>

* deps: repo: make  private

Signed-off-by: Ruslan Kuprieiev <[email protected]>

* update: only support repo imports

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/api.py

@@ -1,6 +1,5 @@
 from contextlib import contextmanager
 import os
-import tempfile
 
 try:
     from contextlib import _GeneratorContextManager as GCM
@@ -9,7 +8,7 @@
 
 from dvc.utils.compat import urlparse
 from dvc.repo import Repo
-from dvc.external_repo import ExternalRepo
+from dvc.external_repo import external_repo
 
 
 def get_url(path, repo=None, rev=None, remote=None):
@@ -68,10 +67,5 @@ def _make_repo(repo_url, rev=None):
         assert rev is None, "Custom revision is not supported for local repo"
         yield Repo(repo_url)
     else:
-        tmp_dir = tempfile.mkdtemp("dvc-repo")
-        ext_repo = ExternalRepo(tmp_dir, url=repo_url, rev=rev)
-        try:
-            ext_repo.install()
-            yield ext_repo.repo
-        finally:
-            ext_repo.uninstall()
+        with external_repo(url=repo_url, rev=rev) as repo:
+            yield repo

dvc/cli.py

@@ -36,6 +36,7 @@
 import dvc.command.tag as tag
 import dvc.command.diff as diff
 import dvc.command.version as version
+import dvc.command.update as update
 from dvc.exceptions import DvcParserError
 
 
@@ -70,6 +71,7 @@
     tag,
     diff,
     version,
+    update,
 ]
 
 

dvc/command/update.py

@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+import argparse
+import logging
+
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase, append_doc_link
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdUpdate(CmdBase):
+    def run(self):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                self.repo.update(target)
+            except DvcException:
+                logger.exception("failed to update '{}'.".format(target))
+                ret = 1
+        return ret
+
+
+def add_parser(subparsers, parent_parser):
+    UPDATE_HELP = "Update dependencies and reproduce specified DVC-files."
+    update_parser = subparsers.add_parser(
+        "update",
+        parents=[parent_parser],
+        description=append_doc_link(UPDATE_HELP, "update"),
+        help=UPDATE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    update_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to update."
+    )
+    update_parser.set_defaults(func=CmdUpdate)

dvc/dependency/__init__.py

@@ -16,7 +16,6 @@
 from .repo import DependencyREPO
 
 from dvc.remote import Remote
-from dvc.external_repo import ExternalRepo
 
 
 DEPS = [
@@ -47,7 +46,7 @@
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)]
 del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
-SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = ExternalRepo.SCHEMA
+SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = DependencyREPO.REPO_SCHEMA
 
 
 def _get(stage, p, info):

dvc/dependency/base.py

@@ -20,3 +20,6 @@ class DependencyBase(object):
 
     DoesNotExistError = DependencyDoesNotExistError
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
+
+    def update(self):
+        raise NotImplementedError

dvc/dependency/repo.py

@@ -1,43 +1,77 @@
 from __future__ import unicode_literals
 
-import os
 import copy
 
-from dvc.utils.compat import urlparse
-from dvc.external_repo import ExternalRepo
+from funcy import merge
+from schema import Optional
+from contextlib import contextmanager
+
+from dvc.external_repo import external_repo
+from dvc.utils.compat import str
 
 from .local import DependencyLOCAL
 
 
 class DependencyREPO(DependencyLOCAL):
     PARAM_REPO = "repo"
+    PARAM_URL = "url"
+    PARAM_REV = "rev"
+    PARAM_REV_LOCK = "rev_lock"
 
-    def __init__(self, erepo, stage, *args, **kwargs):
-        self.erepo = ExternalRepo(stage.repo.dvc_dir, **erepo)
-        super(DependencyLOCAL, self).__init__(stage, *args, **kwargs)
-
-    def _parse_path(self, remote, path):
-        self.erepo.install(self.repo.cache.local.cache_dir)
+    REPO_SCHEMA = {
+        Optional(PARAM_URL): str,
+        Optional(PARAM_REV): str,
+        Optional(PARAM_REV_LOCK): str,
+    }
 
-        out_path = os.path.join(
-            self.erepo.repo.root_dir, urlparse(path).path.lstrip("/")
-        )
+    def __init__(self, def_repo, stage, *args, **kwargs):
+        self.def_repo = def_repo
+        super(DependencyREPO, self).__init__(stage, *args, **kwargs)
 
-        out, = self.erepo.repo.find_outs_by_path(out_path)
-        self.info = copy.copy(out.info)
-        self._erepo_stage = copy.copy(out.stage.path)
-        return self.REMOTE.path_cls(out.cache_path)
+    def _parse_path(self, remote, path):
+        return None
 
     @property
     def is_in_repo(self):
         return False
 
+    def __str__(self):
+        return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
+
+    @contextmanager
+    def _make_repo(self, **overrides):
+        with external_repo(**merge(self.def_repo, overrides)) as repo:
+            yield repo
+
+    def status(self):
+        with self._make_repo() as repo:
+            current = repo.find_out_by_relpath(self.def_path).info
+
+        with self._make_repo(rev_lock=None) as repo:
+            updated = repo.find_out_by_relpath(self.def_path).info
+
+        if current != updated:
+            return {str(self): "update available"}
+
+        return {}
+
+    def save(self):
+        pass
+
     def dumpd(self):
-        ret = super(DependencyLOCAL, self).dumpd()
-        ret[self.PARAM_REPO] = self.erepo.dumpd()
-        return ret
+        return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
     def download(self, to, resume=False):
-        self.erepo.repo.fetch(self._erepo_stage)
-        to.info = copy.copy(self.info)
-        to.checkout()
+        with self._make_repo(
+            cache_dir=self.repo.cache.local.cache_dir
+        ) as repo:
+            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+
+            out = repo.find_out_by_relpath(self.def_path)
+            repo.fetch(out.stage.path)
+            to.info = copy.copy(out.info)
+            to.checkout()
+
+    def update(self):
+        with self._make_repo(rev_lock=None) as repo:
+            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()