From 28d16f7eeeeddd70063a97cfe1e3ef225d890f22 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Wed, 11 Mar 2020 17:16:16 -0700 Subject: [PATCH 1/6] external repo: clone single specified branch with depth 1 (3473 workaround) --- dvc/external_repo.py | 2 +- dvc/scm/git/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index a598eb6692..f5117332b6 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -248,7 +248,7 @@ def _clone_default_branch(url, rev): else: logger.debug("erepo: git clone %s to a temporary dir", url) clone_path = tempfile.mkdtemp("dvc-clone") - git = Git.clone(url, clone_path) + git = Git.clone(url, clone_path, rev=rev) CLONES[url] = clone_path finally: if git: diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index e70d6f7b0f..c822567648 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -76,7 +76,9 @@ def clone(url, to_path, rev=None): url, to_path, env=env, # needed before we can fix it in __init__ - no_single_branch=True, + branch=rev, + single_branch=True, + depth=1, ) tmp_repo.close() except git.exc.GitCommandError as exc: From fd66d76c8cae1ecc1a9426aded7ebd5e49ace6cf Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 2 Apr 2020 16:42:26 +0100 Subject: [PATCH 2/6] git: clone: back to --no-single-branch --- dvc/scm/git/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index c822567648..08d3799418 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -77,7 +77,7 @@ def clone(url, to_path, rev=None): to_path, env=env, # needed before we can fix it in __init__ branch=rev, - single_branch=True, + no_single_branch=True, depth=1, ) tmp_repo.close() From cec5b58c4d3609c9572e2d0f51840edc2155eb05 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 2 Apr 2020 17:22:31 +0100 Subject: [PATCH 3/6] git: clone: try partial, fallback full --- dvc/scm/git/__init__.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 08d3799418..91cb0cb42b 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -1,5 +1,6 @@ """Manages Git.""" +from functools import partial import logging import os import yaml @@ -71,18 +72,26 @@ def clone(url, to_path, rev=None): # [1] https://github.com/gitpython-developers/GitPython/issues/924 env[ld_key] = "" + clone_from = partial( + git.Repo.clone_from, + url, + to_path, + env=env, # needed before we can fix it in __init__ + no_single_branch=True, + ) + try: - tmp_repo = git.Repo.clone_from( - url, - to_path, - env=env, # needed before we can fix it in __init__ - branch=rev, - no_single_branch=True, - depth=1, - ) - tmp_repo.close() + tmp_repo = clone_from(branch=rev, depth=1) except git.exc.GitCommandError as exc: - raise CloneError(url, to_path) from exc + if f"Remote branch {rev} not found" not in str(exc): + raise CloneError(url, to_path) from exc + try: + # not a branch - need to do full clone + tmp_repo = clone_from() + except git.exc.GitCommandError as exc: + raise CloneError(url, to_path) from exc + else: + tmp_repo.close() # NOTE: using our wrapper to make sure that env is fixed in __init__ repo = Git(to_path) From a3cb0cdf3539e5fe429a216f7893baa94dbef8bf Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 2 Apr 2020 17:24:00 +0100 Subject: [PATCH 4/6] git: clone: minor bugfix --- dvc/scm/git/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 91cb0cb42b..185e3ae8eb 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -90,8 +90,8 @@ def clone(url, to_path, rev=None): tmp_repo = clone_from() except git.exc.GitCommandError as exc: raise CloneError(url, to_path) from exc - else: - tmp_repo.close() + + tmp_repo.close() # NOTE: using our wrapper to make sure that env is fixed in __init__ repo = Git(to_path) From e287dc9685961ca07c5077aefbcc0c3aded90441 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 2 Apr 2020 18:54:16 +0100 Subject: [PATCH 5/6] git: clone: shallow only if rev specified --- dvc/scm/git/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 185e3ae8eb..d21d5d24ea 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -81,12 +81,17 @@ def clone(url, to_path, rev=None): ) try: - tmp_repo = clone_from(branch=rev, depth=1) + if rev: + logger.debug("attempting shallow clone of branch %s", rev) + tmp_repo = clone_from(branch=rev, depth=1) + else: + logger.debug("full clone") + tmp_repo = clone_from() except git.exc.GitCommandError as exc: - if f"Remote branch {rev} not found" not in str(exc): + if not rev or f"Remote branch {rev} not found" not in str(exc): raise CloneError(url, to_path) from exc try: - # not a branch - need to do full clone + logger.debug("not a branch - performing full clone") tmp_repo = clone_from() except git.exc.GitCommandError as exc: raise CloneError(url, to_path) from exc From d2666e80450da40e56eee527b20d05ce400d93e0 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 2 Apr 2020 18:58:48 +0100 Subject: [PATCH 6/6] fix py3.5 --- dvc/scm/git/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index d21d5d24ea..2c60f49207 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -88,7 +88,7 @@ def clone(url, to_path, rev=None): logger.debug("full clone") tmp_repo = clone_from() except git.exc.GitCommandError as exc: - if not rev or f"Remote branch {rev} not found" not in str(exc): + if not rev or ("Remote branch %s not found" % rev) not in str(exc): raise CloneError(url, to_path) from exc try: logger.debug("not a branch - performing full clone")