Skip to content

Commit

Permalink
tests: erepo_dir: extract remote setup to separate method (#3748)
Browse files Browse the repository at this point in the history
  • Loading branch information
pared authored May 11, 2020
1 parent 7b4466c commit 451c7aa
Show file tree
Hide file tree
Showing 8 changed files with 97 additions and 100 deletions.
42 changes: 19 additions & 23 deletions tests/dir_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,10 @@
"tmp_dir",
"scm",
"dvc",
"local_remote",
"run_copy",
"erepo_dir",
"git_dir",
"setup_remote",
]


Expand Down Expand Up @@ -256,17 +256,6 @@ def _git_init(path):
git.close()


@pytest.fixture
def local_remote(request, tmp_dir, dvc, make_tmp_dir):
path = make_tmp_dir("local-remote")
with dvc.config.edit() as conf:
conf["remote"]["upstream"] = {"url": fspath(path)}
conf["core"]["remote"] = "upstream"
if "scm" in request.fixturenames:
tmp_dir.scm_add([dvc.config.files["repo"]], commit="add remote")
return path


@pytest.fixture
def run_copy(tmp_dir, dvc):
tmp_dir.gen(
Expand All @@ -287,21 +276,28 @@ def run_copy(src, dst, **run_kwargs):

@pytest.fixture
def erepo_dir(make_tmp_dir):
path = make_tmp_dir("erepo", scm=True, dvc=True)

# Chdir for git and dvc to work locally
with path.chdir():
with path.dvc.config.edit() as conf:
cache_dir = path.dvc.cache.local.cache_dir
conf["remote"]["upstream"] = {"url": cache_dir}
conf["core"]["remote"] = "upstream"
path.scm_add([path.dvc.config.files["repo"]], commit="add remote")

return path
return make_tmp_dir("erepo", scm=True, dvc=True)


@pytest.fixture
def git_dir(make_tmp_dir):
path = make_tmp_dir("git-erepo", scm=True)
path.scm.commit("init repo")
return path


@pytest.fixture
def setup_remote(make_tmp_dir):
def create(repo, url=None, name="upstream", default=True):
if not url:
url = fspath(make_tmp_dir("local_remote"))
with repo.config.edit() as conf:
conf["remote"][name] = {"url": url}
if default:
conf["core"]["remote"] = name

repo.scm.add(repo.config.files["repo"])
repo.scm.commit("add '{}' remote".format(name))
return url

return create
15 changes: 4 additions & 11 deletions tests/func/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ def test_get_url(tmp_dir, dvc, remote_url):


@pytest.mark.parametrize("remote_url", remote_params, indirect=True)
def test_get_url_external(erepo_dir, remote_url):
_set_remote_url_and_commit(erepo_dir.dvc, remote_url)
def test_get_url_external(erepo_dir, remote_url, setup_remote):
setup_remote(erepo_dir.dvc, url=remote_url)
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo", commit="add foo")

Expand Down Expand Up @@ -71,8 +71,8 @@ def test_open(remote_url, tmp_dir, dvc):


@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
def test_open_external(remote_url, erepo_dir):
_set_remote_url_and_commit(erepo_dir.dvc, remote_url)
def test_open_external(remote_url, erepo_dir, setup_remote):
setup_remote(erepo_dir.dvc, url=remote_url)

with erepo_dir.chdir():
erepo_dir.dvc_gen("version", "master", commit="add version")
Expand Down Expand Up @@ -106,13 +106,6 @@ def test_missing(remote_url, tmp_dir, dvc):
api.read("foo")


def _set_remote_url_and_commit(repo, remote_url):
with repo.config.edit() as conf:
conf["remote"]["upstream"]["url"] = remote_url
repo.scm.add([repo.config.files["repo"]])
repo.scm.commit("modify remote")


def test_open_scm_controlled(tmp_dir, erepo_dir):
erepo_dir.scm_gen({"scm_controlled": "file content"}, commit="create file")

Expand Down
49 changes: 30 additions & 19 deletions tests/func/test_data_cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import pytest

from dvc.cache import NamedCache
from dvc.compat import fspath, fspath_py35
from dvc.compat import fspath
from dvc.data_cloud import DataCloud
from dvc.external_repo import clean_repos
from dvc.main import main
Expand Down Expand Up @@ -688,14 +688,13 @@ def test(self):
assert self.message_bar_part in self._caplog.text


def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
def test_verify_checksums(
tmp_dir, scm, dvc, mocker, tmp_path_factory, setup_remote
):

setup_remote(dvc, name="upstream")
tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file")
tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir")

dvc.config["remote"]["local_remote"] = {
"url": fspath(tmp_path_factory.mktemp("local_remote"))
}
dvc.config["core"]["remote"] = "local_remote"
dvc.push()

# remove artifacts and cache to trigger fetching
Expand All @@ -711,7 +710,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory):
# Removing cache will invalidate existing state entries
remove(dvc.cache.local.cache_dir)

dvc.config["remote"]["local_remote"]["verify"] = True
dvc.config["remote"]["upstream"]["verify"] = True

dvc.pull()
assert checksum_spy.call_count == 3
Expand Down Expand Up @@ -784,13 +783,15 @@ def recurse_list_dir(d):
]


def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy):
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, setup_remote):
setup_remote(dvc)
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
stage1 = run_copy("foo", "bar", single_stage=True)
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
dvc.push()

outs = ["foo", "bar", "foobar"]

dvc.push()
clean(outs, dvc)
dvc.pull()
assert all((tmp_dir / file).exists() for file in outs)
Expand All @@ -814,7 +815,8 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy):
assert set(stats["added"]) == set(outs)


def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, setup_remote):
remote_path = setup_remote(dvc)
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "bar", single_stage=True)

Expand All @@ -829,8 +831,11 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
remove(dvc.stage_cache.cache_dir)

dvc.push()

outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]

# each one's a copy of other, hence 3
assert len(recurse_list_dir(fspath_py35(local_remote))) == 3
assert len(recurse_list_dir(remote_path)) == 3

clean(outs, dvc)
assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"}
Expand All @@ -839,13 +844,15 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
assert set(dvc.pull()["added"]) == set(outs)

# clean everything in remote and push
clean(local_remote.iterdir())
from tests.dir_helpers import TmpDir

clean(TmpDir(remote_path).iterdir())
dvc.push(["dvc.yaml:copy-ipsum-baz"])
assert len(recurse_list_dir(fspath_py35(local_remote))) == 1
assert len(recurse_list_dir(remote_path)) == 1

clean(local_remote.iterdir())
clean(TmpDir(remote_path).iterdir())
dvc.push(["dvc.yaml"])
assert len(recurse_list_dir(fspath_py35(local_remote))) == 2
assert len(recurse_list_dir(remote_path)) == 2

with pytest.raises(StageNotFound):
dvc.push(["dvc.yaml:StageThatDoesNotExist"])
Expand All @@ -862,8 +869,10 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy):
({}, "Everything is up to date"),
],
)
def test_push_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
def test_push_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
setup_remote(dvc)
tmp_dir.dvc_gen(fs)

caplog.clear()
with caplog.at_level(level=logging.INFO, logger="dvc"):
main(["push"])
Expand All @@ -878,7 +887,8 @@ def test_push_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
({}, "Everything is up to date."),
],
)
def test_fetch_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog, setup_remote):
setup_remote(dvc)
tmp_dir.dvc_gen(fs)
dvc.push()
clean(list(fs.keys()), dvc)
Expand All @@ -888,7 +898,8 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, local_remote, caplog):
assert msg in caplog.text


def test_pull_stats(tmp_dir, dvc, local_remote, caplog):
def test_pull_stats(tmp_dir, dvc, caplog, setup_remote):
setup_remote(dvc)
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
dvc.push()
clean(["foo", "bar"], dvc)
Expand Down
15 changes: 7 additions & 8 deletions tests/func/test_external_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@ def test_source_change(erepo_dir):
assert old_rev != new_rev


def test_cache_reused(erepo_dir, mocker):
def test_cache_reused(erepo_dir, mocker, setup_remote):
setup_remote(erepo_dir.dvc)
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "text", commit="add file")
erepo_dir.dvc.push()

download_spy = mocker.spy(LocalRemote, "download")

Expand All @@ -63,6 +65,8 @@ def test_cache_reused(erepo_dir, mocker):


def test_known_sha(erepo_dir):
erepo_dir.scm.commit("init")

url = "file://{}".format(erepo_dir)
with external_repo(url) as repo:
rev = repo.scm.get_rev()
Expand Down Expand Up @@ -92,21 +96,16 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
assert dest.read_text() == "contents"


def test_relative_remote(erepo_dir, tmp_dir):
def test_relative_remote(erepo_dir, tmp_dir, setup_remote):
# these steps reproduce the script on this issue:
# https://github.com/iterative/dvc/issues/2756
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")

upstream_dir = tmp_dir
upstream_url = relpath(upstream_dir, erepo_dir)
with erepo_dir.dvc.config.edit() as conf:
conf["remote"]["upstream"] = {"url": upstream_url}
conf["core"]["remote"] = "upstream"
setup_remote(erepo_dir.dvc, url=upstream_url, name="upstream")

erepo_dir.scm_add(
erepo_dir.dvc.config.files["repo"], commit="Update dvc config"
)
erepo_dir.dvc.push()

(erepo_dir / "file").unlink()
Expand Down
27 changes: 11 additions & 16 deletions tests/func/test_gc.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,18 +238,18 @@ def test_gc_without_workspace_raises_error(tmp_dir, dvc):
dvc.gc(force=True, workspace=False)


def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir):
def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir, setup_remote):
dvc = erepo_dir.dvc
with erepo_dir.chdir():
from dvc.exceptions import InvalidArgumentError
setup_remote(dvc)
from dvc.exceptions import InvalidArgumentError

with pytest.raises(InvalidArgumentError):
dvc.gc(force=True, cloud=True)
with pytest.raises(InvalidArgumentError):
dvc.gc(force=True, cloud=True)

dvc.gc(cloud=True, all_tags=True)
dvc.gc(cloud=True, all_commits=True)
dvc.gc(cloud=True, all_branches=True)
dvc.gc(cloud=True, all_commits=False, all_branches=True, all_tags=True)
dvc.gc(cloud=True, all_tags=True)
dvc.gc(cloud=True, all_commits=True)
dvc.gc(cloud=True, all_branches=True)
dvc.gc(cloud=True, all_commits=False, all_branches=True, all_tags=True)


def test_gc_without_workspace_on_tags_branches_commits(tmp_dir, dvc):
Expand Down Expand Up @@ -295,13 +295,8 @@ def test_gc_with_possible_args_positive(tmp_dir, dvc):
assert main(["gc", "-vf", flag]) == 0


def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory):
with dvc.config.edit() as conf:
storage = fspath(tmp_path_factory.mktemp("test_remote_base"))
conf["remote"]["local_remote"] = {"url": storage}
conf["core"]["remote"] = "local_remote"

dvc.push()
def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory, setup_remote):
setup_remote(dvc)

for flag in ["-cw", "-ca", "-cT", "-caT", "-cwT"]:
assert main(["gc", "-vf", flag]) == 0
Expand Down
10 changes: 7 additions & 3 deletions tests/func/test_get.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,11 @@ def test_get_file_from_dir(tmp_dir, erepo_dir):
assert (tmp_dir / "X").read_text() == "foo"


def test_get_url_positive(tmp_dir, erepo_dir, caplog):
def test_get_url_positive(tmp_dir, erepo_dir, caplog, setup_remote):
setup_remote(erepo_dir.dvc)
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo")
erepo_dir.dvc.push()

caplog.clear()
with caplog.at_level(logging.ERROR, logger="dvc"):
Expand All @@ -225,15 +227,17 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog):


def test_get_pipeline_tracked_outs(
tmp_dir, dvc, scm, git_dir, local_remote, run_copy
tmp_dir, dvc, scm, git_dir, run_copy, setup_remote
):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK

setup_remote(dvc)
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
dvc.push()

dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK])
dvc.scm.commit("add pipeline stage")
dvc.push()

with git_dir.chdir():
Repo.get("file:///{}".format(fspath(tmp_dir)), "bar", out="baz")
Expand Down
Loading

0 comments on commit 451c7aa

Please sign in to comment.